{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym, random, pickle, os.path, math, glob\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.autograd as autograd\n",
    "from torch.distributions import Categorical\n",
    "\n",
    "import pdb\n",
    "\n",
    "from atari_wrappers import make_atari, wrap_deepmind,LazyFrames\n",
    "from IPython.display import clear_output\n",
    "from tensorboardX import SummaryWriter\n",
    "\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor\n",
    "Variable = lambda *args, **kwargs: autograd.Variable(*args, **kwargs).cuda() if USE_CUDA else autograd.Variable(*args, **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1fdd4028b08>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2de3RUx53nvz8kJKH3CyRZIIMAE/wIYOMHeLL22vGLPBgPMxmcZI8z8YTNOU7iPGzHjnOym7Mm6+CcjH023nhwkgmZiU0ywQ4exwnxYJvxxCxgjIl5GAsEyBIISaD3AyFR+0fdulWib3ff7nv7cXV/n3N0VF19+3Z9+3Z1/e6vqn4/EkKAYZjJz5RMN4BhmPTAnZ1hQgJ3doYJCdzZGSYkcGdnmJDAnZ1hQoKnzk5EtxPRISI6TEQP+dUohmH8h5KdZyeiHADvA7gFQCuAXQDuEkIc8K95DMP4Ra6H114D4LAQohkAiGgjgJUAonb2nOIikVtZOaFuyqiHFmQJ5/Oc6yeDtsmA0/WZLNfmQm1jZ85gfGCQnI710tnrAXxgPG4FcG2sF+RWVqLuga9OqCv6QN9JTBn30JoM0j/7vGO90hZUXUHnfI78Pzgr8vqUHAuuu0rpAiK1nXz8iaiv86LY6dcj4p6AiNYQ0VtE9Nb4wKCHt2MYxgteRvZWALOMxzMBnLjwICHEegDrAWD25SXikVt/O+H5n39rpV0u6AqmbfWVNb93rFfagqor6IxUSxv3q/dsjnhu45o70t0c31C6gEhta/+xJ+rrvIzsuwDMJ6I5RJQHYDWAFz2cj2GYFJL0yC6EGCOiLwHYAiAHwM+EEPv9aFR/Q75dHs/XdwtF7WMAgKn9Y3bdUK38lRst0b9bBd36PkaNqqPlU/VrpsubnqlD+q6jqO2sXR4rlM8P1OvXkHXfXdY8kqicCShtTroArU3pArQ2J12A1qZ0AVqbky5AayPDn+BWW29jgV0W1imL287ZdblD+qSD9VLvuUKtt7BTPp/Xo19jjlYjFVJvXr/WW9gu9Z4r0V/ZwVpdzjkr9Za0aL3JoLQJ475YaXPSBWhtShegtTnpArQ2pQvQ2px0Ad61eTHjIYR4GcDLnlrAMExaCK5LkmGYhPA0sqeKzqt1WVRq04W2StOpzDDjuxdIe2vkEm2CFu/VZmZBl/zfX6/tsp7l8py5bdoUK2rT7zlaKo89faN+bzEs68qaE9NyIUqbky5Aa1O6AK3NSRegtSldgNbmpAvQ2pQuwL22zuX686dplkn+b/qWxzR3uxZJE3esXrdt/E3ZtirDjO9tNG4xFkm9Be9rvYXt8v/QdP2VNa8PnZHmckmLOw3RUNqULkBrc9IFaG1KF6C1OekCtDalC9DanHQB3rXxyM4wISErR/bGTdEcEZEOpPptw7Kwzawdjjiuav+IUY5+PkA7TeY+Y9aOOR6bKM7aYugCDG2RugCtrWqCezTynKYzSGtLXNe8f3F6jfP04sUvO33GkXU1O4eNsipF6jWdiBMtEW/OK4Vbbd50AbG0pUIXwCM7w4QG7uwMExKy0oxPhNOXSUfH4MzEX1vQpZ0sM3Y7m8iZQukCktM2GShq1WXzNiwb6Lhqml0eqU5856jSZupSayu6Fhnz8X36O1q73dt3lEd2hgkJ3NkZJiQE3oxXc8uPXJ34Qr6nmm7QD3YX+tUkXzDnzJPRNhlYu2uFXa7yZSG2f+TcdNouPzh/W4wjnVHaTF39DXJO/sFVz9t1mzsW2+Xh7XUJv48Jj+wMExICP7KnArWxpP1avfopZ0Q6SibMfydB+zLp2Bkt1U6d6r2RGz4YZ8wNQk6OLK9OrLYb5PUZL9DXp3ZH5MadIMIjO8OEBO7sDBMS2IxnmCyhpEXeLqzbdKddN2GePcpyabfwyM4wIYFHdgeUI6Zhi/8OGa8OpLBjOjAb2mMcmCReHbBeUNpSoQtwMbIT0c+IqIOI9hl1lUT0ChE1Wf8rUtM8hmH8wo0Z/3MAt19Q9xCArUKI+QC2Wo8Zhsli4prxQoj/IKLZF1SvBHCjVd4A4HUA3/SrUSPT9VzqeJ52UOR3y73GZsSQcis6yLoW7dRwi7kRJtpecTeYDhWTi4aSvw0oN6KeJKNtMlDeaj5KfCOM+p44XZ+GJM5nMv5qlV1e9+fEr4/Wlr4NPsk66GqEECcBwPo/w78mMQyTClLujTczwvR3B3sFEsMEmWS98aeIqE4IcZKI6gB0RDvwwowwsU6q9nD3zdOHCaOFNCYf1P1JL2N1DsmUXhq2+G+KOYfRYhJBmfENW/xPtpdt8Q/ckOzI/iKAu63y3QAi8+swDJNVxB3Zieg5SGdcNRG1AvgfAB4D8GsiugdAC4C/8aMxPQvliC4K9S/x9It07qreAblJYbCmyK47fZmsixZKWgX8MyO/OIWSNgMIqs0WJz+hbztUyGWngIRHvuCYIRcNG63MM0bo6+ZV8j1FpZ4vrjZDSVtBB9WGDFObky5Tm1MoaSddpjYzlLRzsMVIDn9Wf21UyOU6I5S0ORd+fIVsmxlKWjkfTevl1DVar1MoaTX/bWaj6bo5MuSyGdBTZVhpWR05ss99xtnIVNrMUNJKm5MuU5vpVFXanHSZ2sx5faXNSdeF2pLBjTf+rihP3ezpnRmGSSu8XJZhQkJWLZcVpdK0LKkYsutGzukm5udHevMrDklza/SENqEKuiPNtpI2XZfzujzWTOxoktcnj616XZ9TJ0CMNHXN40xyhyPNrum75P/xfP2aws7IcypdgNbmpAvQ2nKMdjhpU7rMNtOEU7oz46e/qa+JyMm1zu0801K9V7bjXJOpN1JHWbOuy++Wx+b1Rx5nflbm524mQFTkDkdeR42zU1VpU7oAZ21KF6C1udUFxNYWT1ey8MjOMCEhq0b2ovfkL9rZkryYx+UbA6ZymhTG2TxgRhmJF3FETdmUN7mbsilvcj/15jbtrukMcqvNrS7AvTYnEklbbaaMjoWZgtrMY3chprOzvD+2JUJjclRM5Pq41ea3LkBri6crWXhkZ5iQwJ2dYUJCxs34mq8fscsV52M3p/W5OQCA8sOJr1hTgR4BYM7HZOa8d5t0qpV5G5I3a6c9ejLp1wJaF5Bc5hOlTenKdo7+rhFAcnv7zfUSM+86mnQbhr/tLSzz4bv1+oQr5stdLUoX4E1bPF1lU5Iz83lkZ5iQwJ2dYUJCxs34O6r3xT/I4smixvgHRcGM075yxjsAgNbeMuOI5DPCqPMlixddgNbmtR3pYl3pnPgHRWGsSC9N9qJ3I7yZ8RUz+iPa4UUXoLWl6jryyM4wISHjI7tX1MaSy5Ycs+sOvaF/YZPZftrfIM85Y40+Z/tACQCg7LHiJFqZOEoXoLV51ZXNtNymHW8LPiIdVPv3zLbrvG4CcUvvQwMAgNpiPXJ3rJftcLtGwsRJF6C1pUsXwCM7w4QG7uwMExICb8YrM2h4k3a4eA0mqMw1cy62LNrBKcI075Q2r7qyGfO2ZHiL1NuI9Jm4CnWbNgx9u1bioR1OuoDMaOORnWFCAnd2hgkJbjLCzCKi14joIBHtJ6L7rHrOCsMwAcLNyD4G4BtCiIUArgNwLxFdCs4KwzCBwk0MupMAVEKIfiI6CKAeSWSFOXO8FBvX3JF0Y2s8pqxNFdE0qQ0qAx/Se5pzuuVHPuuPejNDzZA3XcoJtHFL8p9tOvHiaDQDbW7cGQy9blHavOg6czz6ZqiE7tmtNFBLAOyAy6wwZpKI0XODibwdwzA+4nrqjYiKAWwC8FUhRB+Rc/jkCzGTRJSW1PsXUCtLOblcb6U9WyXlTq/ptetO98ocYSc+b6Qe/pHeLskwqcLVyE5EUyE7+i+FEM9b1aesbDCIlxWGYZjM48YbTwB+CuCgEOKHxlOcFYZhAoQbM/56AP8NwLtEpPbefQspygoDAC1f0lFjFtRqg0FFdDGjuWTDRhgzUs1txbq9Lx76sHxNgW7DHTe9Ic89tc+u++llH7fLSlsiG2GcItWoKDxmBB6ly9SmdEXT5oTaLALoDSNqswgwccOIiuiiorkAzpFq3G6EiRap5lC7dBmZt0Sj5TKTS9n9LREaokWqyeRGGKdINUoX4P12z403/j8BRLtB56wwDBMQeAUdw4QEEiJ9DvLSknpx9ZX3pu39TExz6sFVLwAAnmq6wa6rejz5SDXR3ufzK/896nEvtl1hlwvXpnubDeOV0w/orEX3zt8GAFi36U67LlPxBna9/RT6+tscLXEe2RkmJKR1i+t4wRT0zC+If2AUitrlqrOpPmXMqCjUDqKe+ZW+nLPUWMD0s80fdfWauhKtJxltyhE1ND0Y8/UqJ1q8DDZOqDTMADBYm7kd2hWFp30/p9LmRdf4gejjN4/sDBMSuLMzTEhIqx00vaYHX3zghaRf/+RP/goAULPTHzP+0/U79YMHfDllUjzZ/Vd2ORlt7ddK8105HrMd5chq2JK4GX9m4VS7fN/fPx/jyOChtHnRtfb/9UR9jkd2hgkJgY9BN1Qr0zuPlujfrYLu87pspMwNEkoXoLUloqt1VDocd/c0xDxuSfkHdnlOficA4N1BnQOvacBxM6NrriqXq9dm5p2JedxItdY7UiH15vVrvWYK6yDhpAvQ2tKpi0d2hgkJ3NkZJiQE3ozvXiCdUyOX6BVLxXv1XH5BV9qb5AtKF6C1JaJr83G5Qi/aysCOq+TmmX1LLrLrPjy7DQDQOVxk153dVGOXK95LfFXY5gfk6kC1yiwavY1a78Ai+T4F72u9he0Jv3VW4KQL0NrSqYtHdoYJCdzZGSYkBN6Mr99mLXmdYCVmZ2DKRLB1AYY2/3SNLJP7tnOOa5N9ZJb8OpzuM8z4pXrev+I9394+AjOQZI29/CH419FZF5AJbTyyM0xISOvIfnKoDGt3rYh73Oor3rLLcwv8CW1XpAOluGpDOqnuSn8czqJpMkLKd+50WK01Sxe/9updaWqROwqMzyrbrmN5a/xjEuXIiF7nsPHdpXGPPzl0NOpzbmLQFRDRTiLaa2WE+a5VP4eIdlgZYX5FRHnxzsUwTOZwY8afBXCTEGIRgMUAbiei6wB8H8A/WBlhugHck7pmMgzjFTcx6AQAFWFwqvUnANwE4NNW/QYA/xPAj2OdK79TYO4z8U3WfY/quV+/zHgzSGXVfl9O6SPpj2pydruMX39/f+w4obk95lfEnw1IXihrHjHKGWyII/5fx319Rl9w0XfOdEY/xm3c+BwrsmwHgFcAHAHQI4RQV78VMiWU02s5IwzDZAGuHHRCiHEAi4moHMALABY6HRbltSnNCKPC7xp7NyZEiylvkr+2Zhjl7oUyRFdBlw7VNWO3ngpRkV/U1lEAyBmRx06YErMw486ZXPSG3MKZO6TDOauwz6Ol+qOo3hu54cMMmay0OelKhHOl+nIL62d+2l6dwcYM7axJj9VhRjDqk5GmJzhVlWVmbhDqWmRsLOmT18fUMFYor9+Jj+htsYpoMeLabpCfx3iBvj61OyIj66gViAAwUi2PrTioX6PCTjvpMrWZFmeqSWjqTQjRA5nA8ToA5USkvj0zAZzwt2kMw/iJG2/8dGtEBxFNA/BRAAcBvAbgr63DOCMMw2Q5bsz4OgAbiCgH8sfh10KIl4joAICNRPQogD2QKaKYLGOoTt++nLxe1+fVyYwnZ9v0arkPvixdMLP+T+AXVjIOuPHG/xkyTfOF9c0ArklFoxiG8R9eLsswISHw9pryZsabOzeT8pVE5vmbgPK6ug2ImEj2D2ePdyR+rQs4W2YkBymPnZjwynrpIu7E7OTfMEnM2YXypujHmWGcGuLsBVezIA1bxmMfaOA02+KEOXsTC7e60gGP7AwTEtI6sp+dTjjyhWgJYTWrS/2fxTPnrXuWJ556N5VUb9VONHOFmN/cfumByEqdBRo9o9Min88yehv1dey6ObuuY/mb+jr6NX9+udEXNn7hohhHSs5+J3r/4pGdYUICd3aGCQlpNePrCnvxyNUvp/MtbczltJlqQzSe3Kszwvi9uaOkRTsZX998ZeyDrdWe9VkcIWakWpup2XYd17XolM1+bbYyN4K50bu2sDfqczyyM0xICPzUm9q4EC2UtBkDLEgoXYBzKGm3usxNOG6nlTLBqWu0XqdQ0tnc9lg46QK0tnTq4pGdYUICd3aGCQmBN+MrDkkzdfSEnuMs6Ha/YipbUboArS0RXSsvfhcAsPvR2IkdU83K8jjLFS3KmrW2/G6pN68/+NfRSReQGW08sjNMSODOzjAhIfBmvNoYEdTEf9EwN3wko03lQ585I3Ze9GzBzDcf1GScTmSTLh7ZGSYkpHVk7zxVjqcfvzP+gVGobHe35dQtz7bp2Bt9zzoGx00LXnWpgIhPNyf/2aaT2s7knVOVB/Vn5eW75JXST7fZ5U/X74xxpHuUNi+6Ok8di/qc65HdCie9h4hesh5zRhiGCRCJmPH3QQaaVHBGGIYJEK7MeCKaCeBjANYC+DoREZLICJMzcj6peOepontIL2WsyqJ2JYqKrGPGNZ+sTO3XWWnK+zOXoeb0kP97/5U2L7pyRs5Hfc7tyP4EgAcBqDNVgTPCMEygiDuyE9HHAXQIIXYT0Y2q2uFQ3zLCtHxJO3AW1Ootfq3PybAqZhSQ5lVyVdJlS47ZdYfe0OFXEokPp1DZY2as0edsHygBAJQ9Vhxx/LRHTzqep/cHcvWaOeIqbU66AK1N6QK0Nq+6/KL3oQG7XFssQ1J3rJ9t15nx/g7fLbOyXDFfp3c5+juZGsWMx2dm1VnwEZl2eP8efc7GTfKcZsShmXfp9MSH2mVq44Yf6Sw+KrNP2f2Rq/iGv10XU5vSZWozdbnFSRegtSldgNbmpAuYqC0Z3Jjx1wP4JBGtAFAAoBRypC8nolxrdOeMMAyT5cQ144UQDwshZgohZgNYDeBVIcRnwBlhGCZQkMzI7PJgacbfL4T4OBE1AtgIoBIyI8xnhRAx7ZzSknpx9ZX3emhu8pjm1IOrXgAAPNV0g11X9Xhh2tvEBJfTDwzZ5XvnbwMArNuk58czdZu16+2n0Nff5hh1MqFFNUKI1yETO3JGGIYJGLxclmFCAnd2hgkJ3NkZJiSkdSNM5cV9WL3+90m//smfyJDL2RZEMpqmn39rJYCJ2xzdYmbOUSGE1/77Srtu7q8TP2fPfO2k/OIDLyT8+liYmzeSWSV55FN6a8UjH5UTO2t3rbDr5j7j3pGsGKmW5/zc9yInijauuSPh86UaFZzyvr9/PulzHFnVF/U5HtkZJiRwZ2eYkMCdnWFCAnd2hgkJgY9Blw0bYRLB7UaYyYLbjTBuSWQjTDJkciNMquGRnWFCAnd2hgkJgTfjlRk0vEnvT26AN1NYmWvmnucyT2fUKDNzGPrcVR7bm83M2yBvW0y9tR5SQpu3Oeb18SvvjbpNG4a+XStB8qa2eRs5vEW3t9HDOZOFR3aGCQnc2RkmJGTcjP991+V2efR87ObkDia+ZFKR16eXn27uWAwA6O4oseuqEPwkgvHIOas/P/UZpOLcYcD87mwuk5+l+R1LBvX9jndt8qbogJR3VO9zfX4e2RkmJGR8ZD/1w7l2Od6GES+OLHNOd3i7dJTMC8FobmLOE0cLuBiL7gVyzvjWe/9k15Xlys+1+tvv2HWPvvZJuzzvuckZ3lo5HgHtfPTieAS08zHetemuNvKxfM/9yO42bvwxAP0AxgGMCSGWElElgF8BmA3gGIBPCSG6Xb8zwzBpJREz/r8KIRYLIZZajx8CsNXKCLPVeswwTJbixYxfCeBGq7wBMjbdNz22B4BergoA4/na6VHULh0TZlaQoVpp0oyW6N+tgm6dFUPdGqgY4gAwNF3OdU8d0k6lojZt4o4VyucH6vVryLLaypq9zYkrbU66gInaspXNzVfY5XWLNgEAbp2mE4A8cUx/rQbr5XU5V6j1FlqJHZPJYHOuxDh3rS4rB2EyS1pNehvlrYowVt0Wt8l25g5p032wXn9HlbZCI2Gl0jZimNwjFfo7mtcvv6Nmam6lzUkX4F2b25FdAPgjEe0mojVWXY0Q4iQAWP9nOL3QzAjT3z05798YJgi4HdmvF0KcIKIZAF4hovfcvoGZEWb25SWu5mc6rzZeX6l/zWir/DUtM0a/7gXyJ3jkEj3iFu/Vmw8KuuT//nr9U92zXJ4zt03/OhfpDLwYLZXHnr5Rv7cYlnVlzW4UREdpc9IFTNSWbVQckp/xlHOldt13Cz4BALh18UbH13QtkqPeWL3WO/6m1FuVxMg+NF1/Zc3rQ2fkCFoSmfwlITqXy8+fphmj9L9JC88c2ZUuQGtTugCtrbdRf+8GFunvaMH78jta2K7fW2lz0gV41+ZqZBdCnLD+dwB4ATKE9CkiqgMA639H9DMwDJNp4nZ2IioiohJVBnArgH0AXoTMBANwRhiGyXrcmPE1AF6QWZqRC+BZIcQfiGgXgF8T0T0AWgD8jV+Nir7HN9I5Vr/NmtvcZtZGzneaGyiq9kc/H6CdJnOfMWv9Ma+dtQVjI8zxj0nT883P/MCuKyBppn7thM6uU3pUm7uF253WTiSv13SQTryl8mdjybx/cbrOkRouftlJQ2SdGRy1Zqf5TOR3VGlLhS7ARWe3Mr8scqg/DeBm31rCMExK4eWyDBMSMr5c1isqTNHgzMRfW9ClPaozdmdXLHrlrQWAtZDx04uPeQu5ZK41aL828XORZeEue0Mn55ySI+eLqzfpxJiF7YmbnqY2FS/e/AyczN5M0nHVNLs8Up34JqAiK0pXOsOQ8cjOMCEh8CO7mjNXWVMSwUzZjN3ZlbLZdjwChvPR2+imVg4COm21Xzy9RWeESeaTdHZkZddobpJz02m7/OD8bTGOdEZZL9pZnHp4ZGeYkMCdnWFCQlaa8e3LtPNjtFQ7P6r3Rm4eSAXKkWU6sXJGpDNvgnmdBEqbky5AazPjoyvnY6kx/2omT1Sba7oXaoejcj5m2vGoHFmmE6viYOSmFTPpZJ8MK287sQDtyFIbnwCga5GxscSKEpNMLHqTthtke8cLdHtrdyS/cScRlDYnXYB3bTyyM0xI4M7OMCGBOzvDhATu7AwTErLSQefVEeEV5Yhp2OK/Q8atNueNO84oR5fX/c6pwK2D0HQ4ljdFP850zja0Rz8uWbw6YL2gtKVCF8AjO8OEBu7sDBMSstKMnyx0rJYm4bmz/nzMZrgtc3npZODUNXpthRm+yQtT84P1Galgl103O4el8prLnUd2hgkJPLKnkAc//Edfz7e2d6Vdnhj1JPgMzNbRbZLZ1DQZGKmWq+VM/WbeNzMteTK4GtmJqJyIfkNE7xHRQSJaRkSVRPQKETVZ/ys8tYRhmJTi1ox/EsAfhBAfggxRdRCcEYZhAkVcM56ISgH8FwCfAwAhxCiAUSJKWUaYTKM2JJz8hJ5nV3HjnQMSZha1aUbt7Qd0THznwIjp4/gK2TYzbny5ihufxigtbjn8WdklzLjxdVbc+FRvwEo1bkb2RgCdAP6JiPYQ0U+skNKcEYZhAoQbB10ugCsBfFkIsYOInkQCJnsyGWEyTV6f/FWvel1n+CD7hz77RvaSNtm4HKO9Zh67TFK9V7bjXJNum5kTLduY/qbsEiJHd428vvQMUpUH5fs8/biO+jMh15vHsNJuRvZWAK1CiB3W499Adn7OCMMwASJuZxdCtAP4gIgWWFU3AzgAzgjDMIHC7Tz7lwH8kojyADQD+DvIH4qUZITJNCqBX3lT9pqbJmrjTqojqSSDmQo7CHhNye0Fla67PEXJPV11diHEOwCWOjzFGWEYJiAEfgWdmhZ5+vU74xwZScEEJ1byI5DpUEkl9d3n4x8Ug+I2PfL73Wbz3MlQ/6ouP70n9Z9nucf8egXP6jVkTxcm3t66fnUt0zedx2vjGSYkcGdnmJAQeDNerWoqTFF0DzeYUVayGeV4BLLP+VjQNWqUM9gQlwTN8QjwyM4woYE7O8OEhKw041u+pE3MBbV6YV7rc3MATNxA0bxKLsO8bMkxu+7QG3PscsMWeayZZWbOx2RqlXebdJ7neRv0e6oMKzPW6HO2D5QAAMoeK45o77RHTzrq6P1BA4CJ899Km5MuU5vSZWpz0mVqU7pMbU66TG1KVzRtjroeGrDLtcX9AICO9bPtOjPTy+G75QaiK+br9C5HfydTvpjBN1tu01F4FnzkKABg/x59ThWlxcyUM/Ouo3b5ULvcmtHwI53FR2X2Kbs/MhLn8Led94YrbUqXqc1Jl6lN6TK1OekytZnRZ5Q2J10XaksGHtkZJiSQEOnbMDH78hLxyKbFE+p+/i0dfcV00jAM48xItY5L97nvTVylvnbVOzi2r58ufA3AIzvDhAbu7AwTErizM0xI4M7OMCEhrVNvXaNF2NCybEIdfVFPQQVvTRLDpB/T+3Zhf+oajZ4oj0d2hgkJ3NkZJiS4CSW9AMCvjKpGAN8B8AurfjaAYwA+JYTojnWuKU2jmHbb0ViHxKR/9XUAgDOXakPmb1f+BwDgF29fZ9dd8ne7XZ0v59JL7HLLJ6rtcs0tckVUzTS9iurMVy4CAIjdcfInM5OC83+h14OMF8iVa1P7jHUgO99Nd5NcMUVEX6viJgbdISHEYiHEYgBXARgC8AI4SQTDBIpEzfibARwRQhwHsBIyOQSs/3/pZ8MYhvGXRL3xqwE8Z5UnJIkgIsckEX5S3CI3F/zf/73ervvbnV+Qz+3Pd3xNLMYPvG+Xb9xw3C7/6YTccNL8fq1dtwCDCZ+fCS5T/vMdu0zLFgEARK4eG4dWXWuXizbtQBBwPbJbkWU/CeBfE3kDMyPMOZ5cY5iMkcjIfgeAt4UQp6zHp4iozhrVoyaJMDPClFKl77tuCrbLbZm1T7zp2zl7TstzLnyq164b33/It/Mz2c+UD3/ILg/Uyq2nQzP02NizUH+V5zdfBgAQe7LbeZvIPftd0CY8wEkiGCZQuM3PXgjgFgDPG9WPAbiFiJqs5x7zv3kMw/hFWvezl1KluJaSzyuRW1sDABj8hY468/rlvwUAbBootevWX9KIRGl9eLldfumL6wAA5VP0b+Htj3xD1v1ie8LnZoLHifv192G4TsZ4nxR8hosAAAVzSURBVPesXndxPl/fAdObe9PXsDjsEFvRJ87wfnaGCTOBGtkVlK+n2caWS+dIfnOnrjv+gbfzL5HnPF9g/Hpvz55fbyb1TCkq0uXyMgDAWNuJTDXHNTyyMwzDnZ1hwkJWhpKOhzirF+fkvPY2AMDPJLdqvtTRFmIcmbJooV0eL9IBEacMyyszYQ56itxYIq673K6a2nbGLnu9DfOD84ODjuUgwyM7w4QE7uwMExICacYz2cPZFVcDAHr/u56Dnv79yONyLI82ABz8vowjUPm2/voN3KdnhWY9PV2+xrpFY/yBR3aGCQk8sjOeOFsmnW1zK3We5SOLdQQguv00AGDaP+uNJTcuOgAA6PhfetXjwSt03r1TV8lzXvRaChocYnhkZ5iQwJ2dYUICm/GMJypekXHKD85eYNflGc+f3V4FAKh9TccDOHVABu88+vkqu67sgH7NjB/zZqNUwCM7w4SEQG6EYRjGGd4IwzAMd3aGCQtuw1J9jYj2E9E+InqOiAqIaA4R7SCiJiL6lRV9lmGYLCVuZyeiegBfAbBUCHE5gBzI+PHfB/APVkaYbgD3pLKhDMN4w60ZnwtgGhHlAigEcBLATQB+Yz3PGWEYJstxk+utDcAPALRAdvJeALsB9Agh1DbyVgD1qWokwzDecWPGV0DmdZsD4CIARZAJIy7EcQ6PM8IwTHbgxoz/KICjQohOIcQ5yNjxywGUW2Y9AMwE4BiNTwixXgixVAixdCoSz8fGMIw/uOnsLQCuI6JCIiLITK4HALwG4K+tYzgjDMNkOW7u2XdAOuLeBvCu9Zr1AL4J4OtEdBhAFYCfprCdDMN4hJfLMswkgpfLMgzDnZ1hwgJ3doYJCdzZGSYkpNVBR0SdAAYBdMU7NkBUg/VkK5NJC+BOz8VCiOlOT6S1swMAEb0lhFia1jdNIawne5lMWgDvetiMZ5iQwJ2dYUJCJjr7+gy8ZyphPdnLZNICeNST9nt2hmEyA5vxDBMS0trZieh2IjpERIeJ6KF0vrdXiGgWEb1GRAeteHz3WfWVRPSKFYvvFWv/f2Agohwi2kNEL1mPAxtbkIjKieg3RPSedZ2WBfn6+B37MW2dnYhyADwFGfjiUgB3EdGl6Xp/HxgD8A0hxEIA1wG412r/QwC2WrH4tlqPg8R9AA4aj4McW/BJAH8QQnwIwCJIXYG8PimJ/SiESMsfgGUAthiPHwbwcLrePwV6NgO4BcAhAHVWXR2AQ5luWwIaZkJ2gJsAvASAIBdt5Dpds2z+A1AK4CgsP5RRH8jrAxnm7QMAlZAxIF8CcJuX65NOM141XhHYuHVENBvAEgA7ANQIIU4CgPV/RuZaljBPAHgQwHnrcRWCG1uwEUAngH+ybkt+QkRFCOj1ESmI/ZjOzu60xzZwUwFEVAxgE4CvCiH6Mt2eZCGijwPoEELsNqsdDg3KNcoFcCWAHwshlkAuyw6Eye6E19iPTqSzs7cCmGU8jhq3LlshoqmQHf2XQojnrepTRFRnPV8HoCNT7UuQ6wF8koiOAdgIaco/AZexBbOQVgCtQkZWAmR0pSsR3OvjKfajE+ns7LsAzLe8iXmQzoYX0/j+nrDi7/0UwEEhxA+Np16EjMEHBCgWnxDiYSHETCHEbMhr8aoQ4jMIaGxBIUQ7gA+ISOWOVrESA3l9kIrYj2l2OqwA8D6AIwAeybQTJMG2/wWkyfRnAO9Yfysg73O3Amiy/ldmuq1JaLsRwEtWuRHATgCHAfwrgPxMty8BHYsBvGVdo98CqAjy9QHwXQDvAdgH4J8B5Hu5PryCjmFCAq+gY5iQwJ2dYUICd3aGCQnc2RkmJHBnZ5iQwJ2dYUICd3aGCQnc2RkmJPx/osgqrvpcshcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create and wrap the environment\n",
    "env = make_atari('MsPacmanNoFrameskip-v4') # only use in no frameskip environment\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True )\n",
    "n_actions = env.action_space.n\n",
    "state_dim = env.observation_space.shape\n",
    "\n",
    "# env.render()\n",
    "test = env.reset()\n",
    "for i in range(100):\n",
    "    test = env.step(env.action_space.sample())[0]\n",
    "\n",
    "plt.imshow(test._force()[...,0])\n",
    "\n",
    "#plt.imshow(env.render(\"rgb_array\"))\n",
    "# env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class soft_DQN(nn.Module):\n",
    "    def __init__(self, in_channels=4, num_actions=5, REWARD_SCALE = 1):\n",
    "        \"\"\"\n",
    "        Initialize a deep Q-learning network as described in\n",
    "        https://storage.googleapis.com/deepmind-data/assets/papers/DeepMindNature14236Paper.pdf\n",
    "        Arguments:\n",
    "            in_channels: number of channel of input.\n",
    "                i.e The number of most recent frames stacked together as describe in the paper\n",
    "            num_actions: number of action-value to output, one-to-one correspondence to action in game.\n",
    "        \"\"\"\n",
    "        super(soft_DQN, self).__init__()        \n",
    "        self.REWARD_SCALE = REWARD_SCALE\n",
    "        self.conv1 = nn.Conv2d(in_channels, 32, kernel_size=8, stride=4)\n",
    "        self.conv2 = nn.Conv2d(32, 64, kernel_size=4, stride=2)\n",
    "        self.conv3 = nn.Conv2d(64, 64, kernel_size=3, stride=1)\n",
    "        self.fc4 = nn.Linear(7 * 7 * 64, 512)\n",
    "        self.fc5 = nn.Linear(512, num_actions)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.fc4(x.view(x.size(0), -1)))\n",
    "        return self.fc5(x)\n",
    "    \n",
    "    def get_action(self, x):\n",
    "        x = F.relu(self.conv1(x))\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.fc4(x.view(x.size(0), -1)))\n",
    "        action_probs = F.softmax(self.fc5(x)/self.REWARD_SCALE,-1)\n",
    "        action_dist = Categorical(action_probs)\n",
    "        actions = action_dist.sample().view(-1, 1)\n",
    "        return actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Memory_Buffer(object):\n",
    "    def __init__(self, memory_size=1000):\n",
    "        self.buffer = []\n",
    "        self.memory_size = memory_size\n",
    "        self.next_idx = 0\n",
    "        \n",
    "    def push(self, state, action, reward, next_state, done):\n",
    "        data = (state, action, reward, next_state, done)\n",
    "        if len(self.buffer) <= self.memory_size: # buffer not full\n",
    "            self.buffer.append(data)\n",
    "        else: # buffer is full\n",
    "            self.buffer[self.next_idx] = data\n",
    "        self.next_idx = (self.next_idx + 1) % self.memory_size\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        for i in range(batch_size):\n",
    "            idx = random.randint(0, self.size() - 1)\n",
    "            data = self.buffer[idx]\n",
    "            state, action, reward, next_state, done= data\n",
    "            states.append(state)\n",
    "            actions.append(action)\n",
    "            rewards.append(reward)\n",
    "            next_states.append(next_state)\n",
    "            dones.append(done)\n",
    "            \n",
    "            \n",
    "        return np.concatenate(states), actions, rewards, np.concatenate(next_states), dones\n",
    "    \n",
    "    def size(self):\n",
    "        return len(self.buffer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class soft_DQNAgent:\n",
    "    def __init__(self, in_channels = 1, action_space = [], USE_CUDA = False, memory_size = 10000, lr = 1e-4,reward_scale = 1):\n",
    "        self.action_space = action_space\n",
    "        self.memory_buffer = Memory_Buffer(memory_size)\n",
    "        self.alpha = reward_scale\n",
    "        self.DQN = soft_DQN(in_channels = in_channels, num_actions = action_space.n,REWARD_SCALE = reward_scale)\n",
    "        self.DQN_target = soft_DQN(in_channels = in_channels, num_actions = action_space.n,REWARD_SCALE = reward_scale)\n",
    "        self.DQN_target.load_state_dict(self.DQN.state_dict())\n",
    "\n",
    "        self.USE_CUDA = USE_CUDA\n",
    "        if USE_CUDA:\n",
    "            self.DQN = self.DQN.cuda()\n",
    "            self.DQN_target = self.DQN_target.cuda()\n",
    "        self.optimizer = optim.Adam(self.DQN.parameters(),lr=lr)\n",
    "\n",
    "    def observe(self, lazyframe):\n",
    "        # from Lazy frame to tensor\n",
    "        state =  torch.from_numpy(lazyframe._force().transpose(2,0,1)[None]/255).float()\n",
    "        if self.USE_CUDA:\n",
    "            state = state.cuda()\n",
    "        return state\n",
    "\n",
    "    def value(self, state):\n",
    "        q_values = self.DQN(state)\n",
    "        return q_values\n",
    "    \n",
    "    def act(self, state, t, explore_time):\n",
    "        \"\"\"\n",
    "        random policy first,\n",
    "        then sample action according to softmax policy\n",
    "        \"\"\"\n",
    "        if t < explore_time:\n",
    "            action = self.action_space.sample()\n",
    "        else:\n",
    "            # Select a random action (when epsilon probability and q_value are the same in that state)\n",
    "            action = int(self.DQN.get_action(state).squeeze().cpu().detach().numpy())\n",
    "        return action\n",
    "    \n",
    "    def compute_td_loss(self, states, actions, rewards, next_states, is_done, gamma=0.99):\n",
    "        \"\"\" Compute td loss using torch operations only. Use the formula above. \"\"\"\n",
    "        actions = torch.tensor(actions).long()    # shape: [batch_size]\n",
    "        rewards = torch.tensor(rewards, dtype =torch.float)  # shape: [batch_size]\n",
    "        is_done = torch.tensor(is_done).bool()  # shape: [batch_size]\n",
    "        \n",
    "        if self.USE_CUDA:\n",
    "            actions = actions.cuda()\n",
    "            rewards = rewards.cuda()\n",
    "            is_done = is_done.cuda()\n",
    "\n",
    "        # get q-values for all actions in current states\n",
    "        predicted_qvalues = self.DQN(states)\n",
    "\n",
    "        # select q-values for chosen actions\n",
    "        predicted_qvalues_for_actions = predicted_qvalues[\n",
    "          range(states.shape[0]), actions\n",
    "        ]\n",
    "\n",
    "        # compute q-values for all actions in next states\n",
    "        predicted_next_qvalues = self.DQN_target(next_states) # YOUR CODE\n",
    "        # compute V*(next_states) using predicted next q-values\n",
    "        next_state_values =  self.alpha*torch.logsumexp(predicted_next_qvalues/self.alpha, dim = -1) # YOUR CODE        \n",
    "\n",
    "        # compute \"target q-values\" for loss - it's what's inside square parentheses in the above formula.\n",
    "        target_qvalues_for_actions = rewards + gamma*next_state_values # YOUR CODE\n",
    "\n",
    "        # at the last state we shall use simplified formula: Q(s,a) = r(s,a) since s' doesn't exist\n",
    "        target_qvalues_for_actions = torch.where(\n",
    "            is_done, rewards, target_qvalues_for_actions)\n",
    "\n",
    "        # mean squared error loss to minimize\n",
    "        #loss = torch.mean((predicted_qvalues_for_actions -\n",
    "        #                   target_qvalues_for_actions.detach()) ** 2)\n",
    "        loss = F.smooth_l1_loss(predicted_qvalues_for_actions, target_qvalues_for_actions.detach())\n",
    "\n",
    "        return loss\n",
    "    \n",
    "    def sample_from_buffer(self, batch_size):\n",
    "        states, actions, rewards, next_states, dones = [], [], [], [], []\n",
    "        for i in range(batch_size):\n",
    "            idx = random.randint(0, self.memory_buffer.size() - 1)\n",
    "            data = self.memory_buffer.buffer[idx]\n",
    "            frame, action, reward, next_frame, done= data\n",
    "            states.append(self.observe(frame))\n",
    "            actions.append(action)\n",
    "            rewards.append(reward)\n",
    "            next_states.append(self.observe(next_frame))\n",
    "            dones.append(done)\n",
    "        return torch.cat(states), actions, rewards, torch.cat(next_states), dones\n",
    "\n",
    "    def learn_from_experience(self, batch_size):\n",
    "        if self.memory_buffer.size() > batch_size:\n",
    "            states, actions, rewards, next_states, dones = self.sample_from_buffer(batch_size)\n",
    "            td_loss = self.compute_td_loss(states, actions, rewards, next_states, dones)\n",
    "            self.optimizer.zero_grad()\n",
    "            td_loss.backward()\n",
    "            for param in self.DQN.parameters():\n",
    "                param.grad.data.clamp_(-1, 1)\n",
    "\n",
    "            self.optimizer.step()\n",
    "            return(td_loss.item())\n",
    "        else:\n",
    "            return(0)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\softwares\\ANACONDA\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3335: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "F:\\softwares\\ANACONDA\\lib\\site-packages\\numpy\\core\\_methods.py:161: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n",
      "WARNING:root:NaN or Inf found in input tensor.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames:     0, reward:   nan, loss: 0.000000, episode:    0\n",
      "frames:  1000, reward: 240.000000, loss: 0.000000, episode:    2\n",
      "frames:  2000, reward: 227.500000, loss: 0.000000, episode:    4\n",
      "frames:  3000, reward: 218.000000, loss: 0.000000, episode:    5\n",
      "frames:  4000, reward: 260.000000, loss: 0.000000, episode:    7\n",
      "frames:  5000, reward: 244.444444, loss: 0.000000, episode:    9\n",
      "frames:  6000, reward: 239.000000, loss: 0.000000, episode:   11\n",
      "frames:  7000, reward: 237.000000, loss: 0.000000, episode:   14\n",
      "frames:  8000, reward: 209.000000, loss: 0.000000, episode:   16\n",
      "frames:  9000, reward: 202.000000, loss: 0.000000, episode:   18\n",
      "frames: 10000, reward: 222.000000, loss: 1.812364, episode:   20\n",
      "frames: 11000, reward: 239.000000, loss: 0.001745, episode:   22\n",
      "frames: 12000, reward: 327.000000, loss: 0.887473, episode:   24\n",
      "frames: 13000, reward: 350.000000, loss: 0.000939, episode:   26\n",
      "frames: 14000, reward: 372.000000, loss: 0.004092, episode:   28\n",
      "frames: 15000, reward: 444.000000, loss: 0.307884, episode:   30\n",
      "frames: 16000, reward: 438.000000, loss: 0.296039, episode:   32\n",
      "frames: 17000, reward: 377.000000, loss: 0.589913, episode:   33\n",
      "frames: 18000, reward: 414.000000, loss: 0.296373, episode:   35\n",
      "frames: 19000, reward: 463.000000, loss: 0.001728, episode:   36\n",
      "frames: 20000, reward: 469.000000, loss: 1.175363, episode:   38\n",
      "frames: 21000, reward: 395.000000, loss: 0.007204, episode:   41\n",
      "frames: 22000, reward: 392.000000, loss: 0.596335, episode:   42\n",
      "frames: 23000, reward: 494.000000, loss: 0.598028, episode:   44\n",
      "frames: 24000, reward: 463.000000, loss: 0.012387, episode:   46\n",
      "frames: 25000, reward: 435.000000, loss: 0.883645, episode:   48\n",
      "frames: 26000, reward: 415.000000, loss: 0.594272, episode:   49\n",
      "frames: 27000, reward: 534.000000, loss: 1.565327, episode:   51\n",
      "frames: 28000, reward: 434.000000, loss: 0.297978, episode:   53\n",
      "frames: 29000, reward: 440.000000, loss: 1.200785, episode:   54\n",
      "frames: 30000, reward: 488.000000, loss: 1.767203, episode:   55\n",
      "frames: 31000, reward: 521.000000, loss: 0.596592, episode:   57\n",
      "frames: 32000, reward: 576.000000, loss: 1.170380, episode:   59\n",
      "frames: 33000, reward: 472.000000, loss: 0.561846, episode:   61\n",
      "frames: 34000, reward: 632.000000, loss: 0.679475, episode:   63\n",
      "frames: 35000, reward: 625.000000, loss: 0.301223, episode:   64\n",
      "frames: 36000, reward: 733.000000, loss: 0.064624, episode:   65\n",
      "frames: 37000, reward: 781.000000, loss: 0.606289, episode:   67\n",
      "frames: 38000, reward: 762.000000, loss: 0.608214, episode:   69\n",
      "frames: 39000, reward: 771.000000, loss: 0.584496, episode:   70\n",
      "frames: 40000, reward: 650.000000, loss: 0.330459, episode:   72\n",
      "frames: 41000, reward: 721.000000, loss: 0.322125, episode:   74\n",
      "frames: 42000, reward: 514.000000, loss: 1.029922, episode:   76\n",
      "frames: 43000, reward: 498.000000, loss: 0.320428, episode:   77\n",
      "frames: 44000, reward: 488.000000, loss: 0.755640, episode:   79\n",
      "frames: 45000, reward: 556.000000, loss: 0.607751, episode:   81\n",
      "frames: 46000, reward: 529.000000, loss: 0.843837, episode:   83\n",
      "frames: 47000, reward: 478.000000, loss: 2.440101, episode:   84\n",
      "frames: 48000, reward: 519.000000, loss: 0.366725, episode:   85\n",
      "frames: 49000, reward: 610.000000, loss: 1.987633, episode:   87\n",
      "frames: 50000, reward: 625.000000, loss: 1.159403, episode:   88\n",
      "frames: 51000, reward: 543.000000, loss: 0.689304, episode:   90\n",
      "frames: 52000, reward: 611.000000, loss: 2.762293, episode:   92\n",
      "frames: 53000, reward: 645.000000, loss: 0.626095, episode:   93\n",
      "frames: 54000, reward: 596.000000, loss: 0.932479, episode:   95\n",
      "frames: 55000, reward: 736.000000, loss: 0.615605, episode:   96\n",
      "frames: 56000, reward: 644.000000, loss: 1.243769, episode:   98\n",
      "frames: 57000, reward: 698.000000, loss: 1.210147, episode:  100\n",
      "frames: 58000, reward: 645.000000, loss: 0.458458, episode:  102\n",
      "frames: 59000, reward: 670.000000, loss: 0.615642, episode:  103\n",
      "frames: 60000, reward: 679.000000, loss: 1.428812, episode:  104\n",
      "frames: 61000, reward: 563.000000, loss: 0.605627, episode:  106\n",
      "frames: 62000, reward: 573.000000, loss: 0.655698, episode:  108\n",
      "frames: 63000, reward: 637.000000, loss: 0.390077, episode:  110\n",
      "frames: 64000, reward: 634.000000, loss: 0.809561, episode:  112\n",
      "frames: 65000, reward: 629.000000, loss: 0.882654, episode:  113\n",
      "frames: 66000, reward: 643.000000, loss: 1.193377, episode:  115\n",
      "frames: 67000, reward: 634.000000, loss: 0.187480, episode:  116\n",
      "frames: 68000, reward: 721.000000, loss: 0.700815, episode:  118\n",
      "frames: 69000, reward: 590.000000, loss: 1.012649, episode:  121\n",
      "frames: 70000, reward: 620.000000, loss: 0.289243, episode:  122\n",
      "frames: 71000, reward: 547.000000, loss: 1.617675, episode:  124\n",
      "frames: 72000, reward: 561.000000, loss: 0.960173, episode:  125\n",
      "frames: 73000, reward: 498.000000, loss: 0.546116, episode:  127\n",
      "frames: 74000, reward: 602.000000, loss: 0.879176, episode:  128\n",
      "frames: 75000, reward: 624.000000, loss: 0.441851, episode:  130\n",
      "frames: 76000, reward: 746.000000, loss: 0.634285, episode:  131\n",
      "frames: 77000, reward: 737.000000, loss: 1.622151, episode:  132\n",
      "frames: 78000, reward: 748.000000, loss: 0.839090, episode:  133\n",
      "frames: 79000, reward: 822.000000, loss: 0.713790, episode:  135\n",
      "frames: 80000, reward: 852.000000, loss: 0.575982, episode:  136\n",
      "frames: 81000, reward: 802.000000, loss: 0.641957, episode:  138\n",
      "frames: 82000, reward: 819.000000, loss: 1.796752, episode:  139\n",
      "frames: 83000, reward: 710.000000, loss: 1.048610, episode:  141\n",
      "frames: 84000, reward: 634.000000, loss: 6.985494, episode:  143\n",
      "frames: 85000, reward: 616.000000, loss: 0.796257, episode:  145\n",
      "frames: 86000, reward: 660.000000, loss: 0.444912, episode:  146\n",
      "frames: 87000, reward: 593.000000, loss: 0.218039, episode:  148\n",
      "frames: 88000, reward: 582.000000, loss: 1.713902, episode:  150\n",
      "frames: 89000, reward: 567.000000, loss: 0.902461, episode:  151\n",
      "frames: 90000, reward: 640.000000, loss: 0.705128, episode:  153\n",
      "frames: 91000, reward: 545.000000, loss: 1.750994, episode:  155\n",
      "frames: 92000, reward: 584.000000, loss: 0.390896, episode:  156\n",
      "frames: 93000, reward: 713.000000, loss: 0.897154, episode:  158\n",
      "frames: 94000, reward: 716.000000, loss: 0.378982, episode:  160\n",
      "frames: 95000, reward: 762.000000, loss: 1.292235, episode:  161\n",
      "frames: 96000, reward: 817.000000, loss: 0.799887, episode:  163\n",
      "frames: 97000, reward: 811.000000, loss: 0.873831, episode:  164\n",
      "frames: 98000, reward: 745.000000, loss: 1.111742, episode:  166\n",
      "frames: 99000, reward: 618.000000, loss: 0.764024, episode:  168\n",
      "frames: 100000, reward: 679.000000, loss: 0.642402, episode:  170\n",
      "frames: 101000, reward: 696.000000, loss: 1.396178, episode:  171\n",
      "frames: 102000, reward: 710.000000, loss: 0.983700, episode:  172\n",
      "frames: 103000, reward: 698.000000, loss: 0.384634, episode:  174\n",
      "frames: 104000, reward: 680.000000, loss: 0.728033, episode:  175\n",
      "frames: 105000, reward: 707.000000, loss: 0.473365, episode:  177\n",
      "frames: 106000, reward: 695.000000, loss: 1.626721, episode:  179\n",
      "frames: 107000, reward: 720.000000, loss: 0.504899, episode:  180\n",
      "frames: 108000, reward: 656.000000, loss: 1.127313, episode:  182\n",
      "frames: 109000, reward: 641.000000, loss: 0.932335, episode:  184\n",
      "frames: 110000, reward: 675.000000, loss: 2.751688, episode:  185\n",
      "frames: 111000, reward: 634.000000, loss: 3.349427, episode:  187\n",
      "frames: 112000, reward: 639.000000, loss: 1.441974, episode:  189\n",
      "frames: 113000, reward: 629.000000, loss: 0.610006, episode:  190\n",
      "frames: 114000, reward: 600.000000, loss: 0.940323, episode:  192\n",
      "frames: 115000, reward: 603.000000, loss: 0.944452, episode:  193\n",
      "frames: 116000, reward: 642.000000, loss: 0.540959, episode:  195\n",
      "frames: 117000, reward: 694.000000, loss: 0.618074, episode:  196\n",
      "frames: 118000, reward: 650.000000, loss: 1.751933, episode:  198\n",
      "frames: 119000, reward: 679.000000, loss: 1.709333, episode:  200\n",
      "frames: 120000, reward: 768.000000, loss: 0.665514, episode:  201\n",
      "frames: 121000, reward: 808.000000, loss: 0.894098, episode:  203\n",
      "frames: 122000, reward: 785.000000, loss: 1.485702, episode:  204\n",
      "frames: 123000, reward: 680.000000, loss: 2.162747, episode:  206\n",
      "frames: 124000, reward: 713.000000, loss: 1.533499, episode:  207\n",
      "frames: 125000, reward: 723.000000, loss: 1.027895, episode:  209\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 126000, reward: 712.000000, loss: 5.118631, episode:  210\n",
      "frames: 127000, reward: 587.000000, loss: 2.576011, episode:  212\n",
      "frames: 128000, reward: 629.000000, loss: 1.072730, episode:  214\n",
      "frames: 129000, reward: 607.000000, loss: 1.390945, episode:  216\n",
      "frames: 130000, reward: 604.000000, loss: 1.373563, episode:  217\n",
      "frames: 131000, reward: 605.000000, loss: 1.726908, episode:  219\n",
      "frames: 132000, reward: 569.000000, loss: 7.079701, episode:  221\n",
      "frames: 133000, reward: 573.000000, loss: 1.498660, episode:  222\n",
      "frames: 134000, reward: 704.000000, loss: 0.650557, episode:  223\n",
      "frames: 135000, reward: 676.000000, loss: 1.844869, episode:  225\n",
      "frames: 136000, reward: 682.000000, loss: 2.042207, episode:  226\n",
      "frames: 137000, reward: 769.000000, loss: 0.800276, episode:  227\n",
      "frames: 138000, reward: 813.000000, loss: 2.605144, episode:  229\n",
      "frames: 139000, reward: 841.000000, loss: 1.159246, episode:  231\n",
      "frames: 140000, reward: 830.000000, loss: 1.101123, episode:  232\n",
      "frames: 141000, reward: 701.000000, loss: 1.460816, episode:  234\n",
      "frames: 142000, reward: 724.000000, loss: 1.410230, episode:  236\n",
      "frames: 143000, reward: 651.000000, loss: 1.206283, episode:  237\n",
      "frames: 144000, reward: 641.000000, loss: 1.172764, episode:  239\n",
      "frames: 145000, reward: 661.000000, loss: 1.224441, episode:  240\n",
      "frames: 146000, reward: 692.000000, loss: 0.709185, episode:  242\n",
      "frames: 147000, reward: 658.000000, loss: 0.488044, episode:  244\n",
      "frames: 148000, reward: 697.000000, loss: 1.080250, episode:  245\n",
      "frames: 149000, reward: 708.000000, loss: 0.459292, episode:  246\n",
      "frames: 150000, reward: 705.000000, loss: 2.579137, episode:  247\n",
      "frames: 151000, reward: 691.000000, loss: 2.096165, episode:  249\n",
      "frames: 152000, reward: 722.000000, loss: 1.177979, episode:  250\n",
      "frames: 153000, reward: 726.000000, loss: 1.574221, episode:  252\n",
      "frames: 154000, reward: 760.000000, loss: 1.643026, episode:  254\n",
      "frames: 155000, reward: 759.000000, loss: 1.587445, episode:  256\n",
      "frames: 156000, reward: 711.000000, loss: 1.980420, episode:  257\n",
      "frames: 157000, reward: 694.000000, loss: 1.901397, episode:  259\n",
      "frames: 158000, reward: 575.000000, loss: 1.435270, episode:  261\n",
      "frames: 159000, reward: 623.000000, loss: 1.935207, episode:  263\n",
      "frames: 160000, reward: 602.000000, loss: 0.829346, episode:  264\n",
      "frames: 161000, reward: 594.000000, loss: 1.023948, episode:  266\n",
      "frames: 162000, reward: 578.000000, loss: 2.214511, episode:  268\n",
      "frames: 163000, reward: 669.000000, loss: 1.125863, episode:  270\n",
      "frames: 164000, reward: 707.000000, loss: 1.344541, episode:  271\n",
      "frames: 165000, reward: 757.000000, loss: 2.094363, episode:  272\n",
      "frames: 166000, reward: 731.000000, loss: 1.815116, episode:  274\n",
      "frames: 167000, reward: 717.000000, loss: 0.818589, episode:  276\n",
      "frames: 168000, reward: 720.000000, loss: 1.131859, episode:  278\n",
      "frames: 169000, reward: 627.000000, loss: 1.840622, episode:  279\n",
      "frames: 170000, reward: 634.000000, loss: 3.654546, episode:  281\n",
      "frames: 171000, reward: 595.000000, loss: 1.524597, episode:  282\n",
      "frames: 172000, reward: 624.000000, loss: 7.737824, episode:  284\n",
      "frames: 173000, reward: 639.000000, loss: 0.607551, episode:  285\n",
      "frames: 174000, reward: 563.000000, loss: 2.406499, episode:  287\n",
      "frames: 175000, reward: 619.000000, loss: 0.874164, episode:  289\n",
      "frames: 176000, reward: 619.000000, loss: 0.644715, episode:  289\n",
      "frames: 177000, reward: 622.000000, loss: 1.955789, episode:  291\n",
      "frames: 178000, reward: 555.000000, loss: 1.320357, episode:  293\n",
      "frames: 179000, reward: 667.000000, loss: 0.720369, episode:  294\n",
      "frames: 180000, reward: 740.000000, loss: 1.487448, episode:  296\n",
      "frames: 181000, reward: 744.000000, loss: 3.233522, episode:  297\n",
      "frames: 182000, reward: 836.000000, loss: 3.707537, episode:  299\n",
      "frames: 183000, reward: 805.000000, loss: 5.532535, episode:  300\n",
      "frames: 184000, reward: 785.000000, loss: 0.322178, episode:  302\n",
      "frames: 185000, reward: 817.000000, loss: 2.895656, episode:  303\n",
      "frames: 186000, reward: 746.000000, loss: 1.567261, episode:  305\n",
      "frames: 187000, reward: 746.000000, loss: 0.562888, episode:  305\n",
      "frames: 188000, reward: 777.000000, loss: 5.383896, episode:  307\n",
      "frames: 189000, reward: 654.000000, loss: 0.643162, episode:  309\n",
      "frames: 190000, reward: 794.000000, loss: 1.471937, episode:  310\n",
      "frames: 191000, reward: 802.000000, loss: 0.987049, episode:  312\n",
      "frames: 192000, reward: 825.000000, loss: 0.780338, episode:  314\n",
      "frames: 193000, reward: 866.000000, loss: 1.040009, episode:  315\n",
      "frames: 194000, reward: 785.000000, loss: 1.625553, episode:  317\n",
      "frames: 195000, reward: 773.000000, loss: 1.582142, episode:  319\n",
      "frames: 196000, reward: 650.000000, loss: 0.819896, episode:  321\n",
      "frames: 197000, reward: 671.000000, loss: 1.698565, episode:  322\n",
      "frames: 198000, reward: 736.000000, loss: 1.199600, episode:  323\n",
      "frames: 199000, reward: 746.000000, loss: 0.717760, episode:  325\n",
      "frames: 200000, reward: 776.000000, loss: 1.532510, episode:  326\n",
      "frames: 201000, reward: 800.000000, loss: 0.670110, episode:  328\n",
      "frames: 202000, reward: 854.000000, loss: 2.243225, episode:  329\n",
      "frames: 203000, reward: 846.000000, loss: 0.545529, episode:  331\n",
      "frames: 204000, reward: 888.000000, loss: 3.319028, episode:  332\n",
      "frames: 205000, reward: 738.000000, loss: 1.557017, episode:  334\n",
      "frames: 206000, reward: 716.000000, loss: 0.980858, episode:  336\n",
      "frames: 207000, reward: 795.000000, loss: 0.753246, episode:  338\n",
      "frames: 208000, reward: 769.000000, loss: 1.646408, episode:  339\n",
      "frames: 209000, reward: 853.000000, loss: 1.489250, episode:  341\n",
      "frames: 210000, reward: 807.000000, loss: 2.042443, episode:  342\n",
      "frames: 211000, reward: 891.000000, loss: 1.027675, episode:  344\n",
      "frames: 212000, reward: 848.000000, loss: 1.796908, episode:  345\n",
      "frames: 213000, reward: 792.000000, loss: 1.569473, episode:  347\n",
      "frames: 214000, reward: 865.000000, loss: 1.401230, episode:  348\n",
      "frames: 215000, reward: 760.000000, loss: 8.239844, episode:  350\n",
      "frames: 216000, reward: 735.000000, loss: 0.684218, episode:  351\n",
      "frames: 217000, reward: 710.000000, loss: 0.842105, episode:  353\n",
      "frames: 218000, reward: 661.000000, loss: 1.071492, episode:  354\n",
      "frames: 219000, reward: 668.000000, loss: 1.476798, episode:  356\n",
      "frames: 220000, reward: 624.000000, loss: 0.870404, episode:  358\n",
      "frames: 221000, reward: 630.000000, loss: 0.924149, episode:  359\n",
      "frames: 222000, reward: 669.000000, loss: 1.025653, episode:  360\n",
      "frames: 223000, reward: 699.000000, loss: 1.846929, episode:  362\n",
      "frames: 224000, reward: 762.000000, loss: 1.345154, episode:  363\n",
      "frames: 225000, reward: 764.000000, loss: 0.535654, episode:  365\n",
      "frames: 226000, reward: 877.000000, loss: 1.234875, episode:  366\n",
      "frames: 227000, reward: 842.000000, loss: 1.249645, episode:  367\n",
      "frames: 228000, reward: 800.000000, loss: 0.940310, episode:  369\n",
      "frames: 229000, reward: 823.000000, loss: 0.926806, episode:  370\n",
      "frames: 230000, reward: 929.000000, loss: 1.253765, episode:  372\n",
      "frames: 231000, reward: 866.000000, loss: 0.706536, episode:  373\n",
      "frames: 232000, reward: 839.000000, loss: 0.905443, episode:  375\n",
      "frames: 233000, reward: 754.000000, loss: 1.664465, episode:  376\n",
      "frames: 234000, reward: 778.000000, loss: 0.923175, episode:  377\n",
      "frames: 235000, reward: 846.000000, loss: 0.692228, episode:  379\n",
      "frames: 236000, reward: 750.000000, loss: 0.762062, episode:  381\n",
      "frames: 237000, reward: 642.000000, loss: 1.839404, episode:  382\n",
      "frames: 238000, reward: 729.000000, loss: 0.886138, episode:  384\n",
      "frames: 239000, reward: 707.000000, loss: 1.128560, episode:  386\n",
      "frames: 240000, reward: 748.000000, loss: 1.901304, episode:  387\n",
      "frames: 241000, reward: 669.000000, loss: 1.731673, episode:  389\n",
      "frames: 242000, reward: 717.000000, loss: 1.401533, episode:  390\n",
      "frames: 243000, reward: 714.000000, loss: 5.059178, episode:  392\n",
      "frames: 244000, reward: 654.000000, loss: 1.370666, episode:  394\n",
      "frames: 245000, reward: 658.000000, loss: 1.710061, episode:  395\n",
      "frames: 246000, reward: 600.000000, loss: 1.783482, episode:  397\n",
      "frames: 247000, reward: 600.000000, loss: 8.541126, episode:  399\n",
      "frames: 248000, reward: 559.000000, loss: 3.097463, episode:  401\n",
      "frames: 249000, reward: 601.000000, loss: 1.289278, episode:  402\n",
      "frames: 250000, reward: 621.000000, loss: 5.105525, episode:  404\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 251000, reward: 616.000000, loss: 0.852252, episode:  406\n",
      "frames: 252000, reward: 631.000000, loss: 1.609905, episode:  408\n",
      "frames: 253000, reward: 662.000000, loss: 0.896999, episode:  409\n",
      "frames: 254000, reward: 631.000000, loss: 2.759434, episode:  412\n",
      "frames: 255000, reward: 653.000000, loss: 1.065663, episode:  413\n",
      "frames: 256000, reward: 757.000000, loss: 2.047700, episode:  414\n",
      "frames: 257000, reward: 801.000000, loss: 2.111063, episode:  416\n",
      "frames: 258000, reward: 870.000000, loss: 5.249817, episode:  418\n",
      "frames: 259000, reward: 902.000000, loss: 4.471899, episode:  419\n",
      "frames: 260000, reward: 920.000000, loss: 0.775521, episode:  420\n",
      "frames: 261000, reward: 1137.000000, loss: 1.181427, episode:  422\n",
      "frames: 262000, reward: 1131.000000, loss: 1.775899, episode:  423\n",
      "frames: 263000, reward: 971.000000, loss: 1.520790, episode:  425\n",
      "frames: 264000, reward: 957.000000, loss: 1.219524, episode:  426\n",
      "frames: 265000, reward: 1007.000000, loss: 3.346798, episode:  428\n",
      "frames: 266000, reward: 948.000000, loss: 0.869667, episode:  430\n",
      "frames: 267000, reward: 793.000000, loss: 7.185538, episode:  432\n",
      "frames: 268000, reward: 828.000000, loss: 3.178205, episode:  433\n",
      "frames: 269000, reward: 947.000000, loss: 7.000409, episode:  434\n",
      "frames: 270000, reward: 945.000000, loss: 1.719509, episode:  435\n",
      "frames: 271000, reward: 883.000000, loss: 1.111959, episode:  437\n",
      "frames: 272000, reward: 841.000000, loss: 4.225729, episode:  439\n",
      "frames: 273000, reward: 910.000000, loss: 2.059360, episode:  440\n",
      "frames: 274000, reward: 860.000000, loss: 2.437427, episode:  441\n",
      "frames: 275000, reward: 921.000000, loss: 1.042203, episode:  442\n",
      "frames: 276000, reward: 827.000000, loss: 0.712548, episode:  444\n",
      "frames: 277000, reward: 800.000000, loss: 1.137625, episode:  446\n",
      "frames: 278000, reward: 805.000000, loss: 1.536894, episode:  448\n",
      "frames: 279000, reward: 844.000000, loss: 2.657606, episode:  449\n",
      "frames: 280000, reward: 818.000000, loss: 1.663638, episode:  451\n",
      "frames: 281000, reward: 854.000000, loss: 2.255301, episode:  452\n",
      "frames: 282000, reward: 814.000000, loss: 1.649859, episode:  454\n",
      "frames: 283000, reward: 836.000000, loss: 0.993956, episode:  455\n",
      "frames: 284000, reward: 831.000000, loss: 2.356778, episode:  457\n",
      "frames: 285000, reward: 977.000000, loss: 1.579568, episode:  459\n",
      "frames: 286000, reward: 1004.000000, loss: 1.207605, episode:  460\n",
      "frames: 287000, reward: 1026.000000, loss: 1.412761, episode:  461\n",
      "frames: 288000, reward: 1045.000000, loss: 1.104816, episode:  462\n",
      "frames: 289000, reward: 992.000000, loss: 1.008197, episode:  464\n",
      "frames: 290000, reward: 997.000000, loss: 8.687877, episode:  465\n",
      "frames: 291000, reward: 1046.000000, loss: 2.302245, episode:  466\n",
      "frames: 292000, reward: 964.000000, loss: 0.827459, episode:  468\n",
      "frames: 293000, reward: 851.000000, loss: 1.044337, episode:  470\n",
      "frames: 294000, reward: 715.000000, loss: 0.970080, episode:  472\n",
      "frames: 295000, reward: 710.000000, loss: 1.217027, episode:  474\n",
      "frames: 296000, reward: 628.000000, loss: 1.807336, episode:  476\n",
      "frames: 297000, reward: 563.000000, loss: 4.985443, episode:  477\n",
      "frames: 298000, reward: 680.000000, loss: 0.859545, episode:  479\n",
      "frames: 299000, reward: 702.000000, loss: 1.651606, episode:  481\n",
      "frames: 300000, reward: 672.000000, loss: 7.704388, episode:  483\n",
      "frames: 301000, reward: 659.000000, loss: 0.846911, episode:  484\n",
      "frames: 302000, reward: 704.000000, loss: 1.227288, episode:  486\n",
      "frames: 303000, reward: 667.000000, loss: 5.959506, episode:  488\n",
      "frames: 304000, reward: 600.000000, loss: 1.610615, episode:  490\n",
      "frames: 305000, reward: 584.000000, loss: 0.550101, episode:  492\n",
      "frames: 306000, reward: 682.000000, loss: 1.410671, episode:  493\n",
      "frames: 307000, reward: 670.000000, loss: 0.973351, episode:  495\n",
      "frames: 308000, reward: 707.000000, loss: 3.689111, episode:  497\n",
      "frames: 309000, reward: 737.000000, loss: 2.192552, episode:  499\n",
      "frames: 310000, reward: 758.000000, loss: 1.059960, episode:  500\n",
      "frames: 311000, reward: 818.000000, loss: 0.995186, episode:  502\n",
      "frames: 312000, reward: 694.000000, loss: 2.390296, episode:  504\n",
      "frames: 313000, reward: 709.000000, loss: 2.485892, episode:  506\n",
      "frames: 314000, reward: 747.000000, loss: 0.746425, episode:  507\n",
      "frames: 315000, reward: 748.000000, loss: 1.406046, episode:  508\n",
      "frames: 316000, reward: 823.000000, loss: 1.060823, episode:  509\n",
      "frames: 317000, reward: 989.000000, loss: 1.068237, episode:  511\n",
      "frames: 318000, reward: 1038.000000, loss: 1.043980, episode:  513\n",
      "frames: 319000, reward: 1072.000000, loss: 5.756521, episode:  515\n",
      "frames: 320000, reward: 1045.000000, loss: 5.006078, episode:  516\n",
      "frames: 321000, reward: 1021.000000, loss: 1.336449, episode:  517\n",
      "frames: 322000, reward: 919.000000, loss: 1.811872, episode:  519\n",
      "frames: 323000, reward: 690.000000, loss: 1.412592, episode:  521\n",
      "frames: 324000, reward: 652.000000, loss: 1.483793, episode:  523\n",
      "frames: 325000, reward: 690.000000, loss: 2.430918, episode:  524\n",
      "frames: 326000, reward: 743.000000, loss: 2.525680, episode:  526\n",
      "frames: 327000, reward: 688.000000, loss: 4.144138, episode:  528\n",
      "frames: 328000, reward: 686.000000, loss: 1.934126, episode:  530\n",
      "frames: 329000, reward: 707.000000, loss: 1.294499, episode:  531\n",
      "frames: 330000, reward: 885.000000, loss: 0.994898, episode:  533\n",
      "frames: 331000, reward: 819.000000, loss: 1.688892, episode:  534\n",
      "frames: 332000, reward: 862.000000, loss: 2.249414, episode:  535\n",
      "frames: 333000, reward: 943.000000, loss: 1.972883, episode:  536\n",
      "frames: 334000, reward: 1210.000000, loss: 1.412745, episode:  538\n",
      "frames: 335000, reward: 1210.000000, loss: 1.309145, episode:  538\n",
      "frames: 336000, reward: 1248.000000, loss: 2.804306, episode:  540\n",
      "frames: 337000, reward: 1215.000000, loss: 1.784314, episode:  542\n",
      "frames: 338000, reward: 1087.000000, loss: 2.211472, episode:  543\n",
      "frames: 339000, reward: 1111.000000, loss: 4.119308, episode:  544\n",
      "frames: 340000, reward: 972.000000, loss: 1.410685, episode:  546\n",
      "frames: 341000, reward: 763.000000, loss: 1.447398, episode:  548\n",
      "frames: 342000, reward: 679.000000, loss: 2.862203, episode:  550\n",
      "frames: 343000, reward: 718.000000, loss: 7.206176, episode:  552\n",
      "frames: 344000, reward: 711.000000, loss: 1.645199, episode:  553\n",
      "frames: 345000, reward: 785.000000, loss: 1.438175, episode:  554\n",
      "frames: 346000, reward: 778.000000, loss: 1.329906, episode:  556\n",
      "frames: 347000, reward: 841.000000, loss: 2.195586, episode:  558\n",
      "frames: 348000, reward: 820.000000, loss: 5.718067, episode:  559\n",
      "frames: 349000, reward: 872.000000, loss: 1.521141, episode:  561\n",
      "frames: 350000, reward: 943.000000, loss: 1.086118, episode:  562\n",
      "frames: 351000, reward: 978.000000, loss: 0.732535, episode:  563\n",
      "frames: 352000, reward: 842.000000, loss: 1.533530, episode:  565\n",
      "frames: 353000, reward: 840.000000, loss: 0.946787, episode:  566\n",
      "frames: 354000, reward: 805.000000, loss: 2.456646, episode:  568\n",
      "frames: 355000, reward: 799.000000, loss: 3.037867, episode:  569\n",
      "frames: 356000, reward: 776.000000, loss: 1.883469, episode:  571\n",
      "frames: 357000, reward: 631.000000, loss: 1.007904, episode:  573\n",
      "frames: 358000, reward: 748.000000, loss: 1.667208, episode:  574\n",
      "frames: 359000, reward: 780.000000, loss: 1.039220, episode:  576\n",
      "frames: 360000, reward: 791.000000, loss: 1.785945, episode:  577\n",
      "frames: 361000, reward: 830.000000, loss: 2.046815, episode:  578\n",
      "frames: 362000, reward: 900.000000, loss: 2.071733, episode:  580\n",
      "frames: 363000, reward: 927.000000, loss: 1.594009, episode:  582\n",
      "frames: 364000, reward: 1085.000000, loss: 1.207615, episode:  584\n",
      "frames: 365000, reward: 1063.000000, loss: 0.638726, episode:  585\n",
      "frames: 366000, reward: 1090.000000, loss: 2.023026, episode:  587\n",
      "frames: 367000, reward: 955.000000, loss: 1.063135, episode:  589\n",
      "frames: 368000, reward: 1042.000000, loss: 1.742923, episode:  590\n",
      "frames: 369000, reward: 1001.000000, loss: 6.961444, episode:  591\n",
      "frames: 370000, reward: 1071.000000, loss: 1.804876, episode:  593\n",
      "frames: 371000, reward: 949.000000, loss: 5.792548, episode:  595\n",
      "frames: 372000, reward: 826.000000, loss: 1.403526, episode:  597\n",
      "frames: 373000, reward: 778.000000, loss: 1.940284, episode:  599\n",
      "frames: 374000, reward: 729.000000, loss: 16.280943, episode:  601\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 375000, reward: 717.000000, loss: 1.886225, episode:  602\n",
      "frames: 376000, reward: 704.000000, loss: 7.676212, episode:  604\n",
      "frames: 377000, reward: 765.000000, loss: 2.063270, episode:  605\n",
      "frames: 378000, reward: 806.000000, loss: 3.479015, episode:  607\n",
      "frames: 379000, reward: 970.000000, loss: 1.666684, episode:  609\n",
      "frames: 380000, reward: 958.000000, loss: 0.876122, episode:  611\n",
      "frames: 381000, reward: 769.000000, loss: 1.073261, episode:  613\n",
      "frames: 382000, reward: 861.000000, loss: 2.360525, episode:  614\n",
      "frames: 383000, reward: 776.000000, loss: 1.948852, episode:  616\n",
      "frames: 384000, reward: 616.000000, loss: 0.791491, episode:  618\n",
      "frames: 385000, reward: 652.000000, loss: 1.588701, episode:  620\n",
      "frames: 386000, reward: 669.000000, loss: 7.423689, episode:  621\n",
      "frames: 387000, reward: 743.000000, loss: 1.002911, episode:  623\n",
      "frames: 388000, reward: 674.000000, loss: 2.184133, episode:  624\n",
      "frames: 389000, reward: 774.000000, loss: 1.261963, episode:  626\n",
      "frames: 390000, reward: 781.000000, loss: 1.421649, episode:  627\n",
      "frames: 391000, reward: 936.000000, loss: 2.185104, episode:  629\n",
      "frames: 392000, reward: 1003.000000, loss: 2.289899, episode:  631\n",
      "frames: 393000, reward: 987.000000, loss: 3.378642, episode:  633\n",
      "frames: 394000, reward: 1034.000000, loss: 1.587477, episode:  634\n",
      "frames: 395000, reward: 943.000000, loss: 1.103833, episode:  636\n",
      "frames: 396000, reward: 750.000000, loss: 2.258782, episode:  638\n",
      "frames: 397000, reward: 754.000000, loss: 1.962951, episode:  640\n",
      "frames: 398000, reward: 802.000000, loss: 1.197044, episode:  641\n",
      "frames: 399000, reward: 797.000000, loss: 7.107166, episode:  643\n",
      "frames: 400000, reward: 818.000000, loss: 4.627372, episode:  645\n",
      "frames: 401000, reward: 802.000000, loss: 2.644910, episode:  647\n",
      "frames: 402000, reward: 792.000000, loss: 1.986018, episode:  649\n",
      "frames: 403000, reward: 635.000000, loss: 1.450588, episode:  651\n",
      "frames: 404000, reward: 612.000000, loss: 1.163276, episode:  653\n",
      "frames: 405000, reward: 523.000000, loss: 2.620439, episode:  655\n",
      "frames: 406000, reward: 532.000000, loss: 2.880766, episode:  657\n",
      "frames: 407000, reward: 579.000000, loss: 4.205995, episode:  658\n",
      "frames: 408000, reward: 716.000000, loss: 1.211119, episode:  660\n",
      "frames: 409000, reward: 692.000000, loss: 1.501320, episode:  662\n",
      "frames: 410000, reward: 727.000000, loss: 1.350076, episode:  663\n",
      "frames: 411000, reward: 774.000000, loss: 1.830734, episode:  665\n",
      "frames: 412000, reward: 789.000000, loss: 1.139515, episode:  667\n",
      "frames: 413000, reward: 924.000000, loss: 4.295081, episode:  668\n",
      "frames: 414000, reward: 836.000000, loss: 2.351724, episode:  670\n",
      "frames: 415000, reward: 821.000000, loss: 2.075819, episode:  672\n",
      "frames: 416000, reward: 786.000000, loss: 1.285085, episode:  674\n",
      "frames: 417000, reward: 841.000000, loss: 1.143946, episode:  676\n",
      "frames: 418000, reward: 854.000000, loss: 3.693803, episode:  677\n",
      "frames: 419000, reward: 680.000000, loss: 1.047035, episode:  678\n",
      "frames: 420000, reward: 692.000000, loss: 2.167265, episode:  680\n",
      "frames: 421000, reward: 710.000000, loss: 1.641425, episode:  682\n",
      "frames: 422000, reward: 694.000000, loss: 7.523617, episode:  684\n",
      "frames: 423000, reward: 724.000000, loss: 3.589808, episode:  686\n",
      "frames: 424000, reward: 693.000000, loss: 2.743671, episode:  687\n",
      "frames: 425000, reward: 707.000000, loss: 1.345653, episode:  689\n",
      "frames: 426000, reward: 680.000000, loss: 1.663754, episode:  691\n",
      "frames: 427000, reward: 723.000000, loss: 1.032643, episode:  693\n",
      "frames: 428000, reward: 657.000000, loss: 2.579074, episode:  695\n",
      "frames: 429000, reward: 674.000000, loss: 1.213296, episode:  696\n",
      "frames: 430000, reward: 646.000000, loss: 2.557038, episode:  698\n",
      "frames: 431000, reward: 716.000000, loss: 1.960402, episode:  700\n",
      "frames: 432000, reward: 729.000000, loss: 2.255334, episode:  702\n",
      "frames: 433000, reward: 662.000000, loss: 1.130264, episode:  704\n",
      "frames: 434000, reward: 642.000000, loss: 2.482434, episode:  706\n",
      "frames: 435000, reward: 792.000000, loss: 4.525570, episode:  707\n",
      "frames: 436000, reward: 775.000000, loss: 6.897086, episode:  709\n",
      "frames: 437000, reward: 731.000000, loss: 2.751349, episode:  711\n",
      "frames: 438000, reward: 762.000000, loss: 1.307539, episode:  713\n",
      "frames: 439000, reward: 853.000000, loss: 3.207136, episode:  715\n",
      "frames: 440000, reward: 852.000000, loss: 2.489125, episode:  716\n",
      "frames: 441000, reward: 743.000000, loss: 7.925100, episode:  718\n",
      "frames: 442000, reward: 770.000000, loss: 1.785297, episode:  720\n",
      "frames: 443000, reward: 790.000000, loss: 1.964388, episode:  721\n",
      "frames: 444000, reward: 763.000000, loss: 1.880454, episode:  723\n",
      "frames: 445000, reward: 693.000000, loss: 8.617126, episode:  725\n",
      "frames: 446000, reward: 682.000000, loss: 1.672978, episode:  727\n",
      "frames: 447000, reward: 669.000000, loss: 1.970657, episode:  729\n",
      "frames: 448000, reward: 713.000000, loss: 3.348972, episode:  730\n",
      "frames: 449000, reward: 710.000000, loss: 0.890532, episode:  731\n",
      "frames: 450000, reward: 759.000000, loss: 1.501886, episode:  733\n",
      "frames: 451000, reward: 789.000000, loss: 2.355804, episode:  734\n",
      "frames: 452000, reward: 784.000000, loss: 1.214685, episode:  736\n",
      "frames: 453000, reward: 812.000000, loss: 4.722915, episode:  738\n",
      "frames: 454000, reward: 757.000000, loss: 2.129539, episode:  740\n",
      "frames: 455000, reward: 771.000000, loss: 1.574612, episode:  741\n",
      "frames: 456000, reward: 776.000000, loss: 7.034272, episode:  743\n",
      "frames: 457000, reward: 762.000000, loss: 1.165489, episode:  745\n",
      "frames: 458000, reward: 775.000000, loss: 1.615662, episode:  746\n",
      "frames: 459000, reward: 832.000000, loss: 3.118916, episode:  747\n",
      "frames: 460000, reward: 825.000000, loss: 1.550367, episode:  749\n",
      "frames: 461000, reward: 848.000000, loss: 6.284337, episode:  751\n",
      "frames: 462000, reward: 890.000000, loss: 3.146451, episode:  752\n",
      "frames: 463000, reward: 896.000000, loss: 2.357586, episode:  754\n",
      "frames: 464000, reward: 905.000000, loss: 1.001972, episode:  755\n",
      "frames: 465000, reward: 875.000000, loss: 3.234801, episode:  757\n",
      "frames: 466000, reward: 871.000000, loss: 2.618709, episode:  759\n",
      "frames: 467000, reward: 898.000000, loss: 4.888561, episode:  761\n",
      "frames: 468000, reward: 823.000000, loss: 2.909209, episode:  763\n",
      "frames: 469000, reward: 767.000000, loss: 3.753940, episode:  764\n",
      "frames: 470000, reward: 689.000000, loss: 1.925842, episode:  766\n",
      "frames: 471000, reward: 698.000000, loss: 1.795009, episode:  768\n",
      "frames: 472000, reward: 722.000000, loss: 1.321571, episode:  769\n",
      "frames: 473000, reward: 687.000000, loss: 1.831632, episode:  771\n",
      "frames: 474000, reward: 696.000000, loss: 1.927233, episode:  774\n",
      "frames: 475000, reward: 698.000000, loss: 1.438538, episode:  775\n",
      "frames: 476000, reward: 718.000000, loss: 4.221125, episode:  776\n",
      "frames: 477000, reward: 738.000000, loss: 1.158021, episode:  778\n",
      "frames: 478000, reward: 695.000000, loss: 1.780053, episode:  780\n",
      "frames: 479000, reward: 693.000000, loss: 1.834548, episode:  781\n",
      "frames: 480000, reward: 704.000000, loss: 0.955455, episode:  784\n",
      "frames: 481000, reward: 622.000000, loss: 1.539664, episode:  786\n",
      "frames: 482000, reward: 711.000000, loss: 2.158676, episode:  787\n",
      "frames: 483000, reward: 715.000000, loss: 2.773340, episode:  789\n",
      "frames: 484000, reward: 750.000000, loss: 1.047698, episode:  790\n",
      "frames: 485000, reward: 707.000000, loss: 1.621632, episode:  793\n",
      "frames: 486000, reward: 751.000000, loss: 6.834707, episode:  794\n",
      "frames: 487000, reward: 753.000000, loss: 1.262048, episode:  796\n",
      "frames: 488000, reward: 682.000000, loss: 8.266472, episode:  798\n",
      "frames: 489000, reward: 692.000000, loss: 1.367098, episode:  799\n",
      "frames: 490000, reward: 682.000000, loss: 0.830594, episode:  801\n",
      "frames: 491000, reward: 777.000000, loss: 2.263378, episode:  802\n",
      "frames: 492000, reward: 834.000000, loss: 1.828416, episode:  804\n",
      "frames: 493000, reward: 887.000000, loss: 1.869537, episode:  806\n",
      "frames: 494000, reward: 888.000000, loss: 4.846390, episode:  807\n",
      "frames: 495000, reward: 977.000000, loss: 7.057942, episode:  809\n",
      "frames: 496000, reward: 945.000000, loss: 1.399500, episode:  811\n",
      "frames: 497000, reward: 760.000000, loss: 1.221840, episode:  813\n",
      "frames: 498000, reward: 755.000000, loss: 1.930882, episode:  815\n",
      "frames: 499000, reward: 778.000000, loss: 2.021456, episode:  817\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 500000, reward: 731.000000, loss: 1.852991, episode:  818\n",
      "frames: 501000, reward: 651.000000, loss: 1.284324, episode:  820\n",
      "frames: 502000, reward: 732.000000, loss: 1.703142, episode:  821\n",
      "frames: 503000, reward: 920.000000, loss: 2.428574, episode:  822\n",
      "frames: 504000, reward: 910.000000, loss: 0.975000, episode:  824\n",
      "frames: 505000, reward: 998.000000, loss: 2.203589, episode:  826\n",
      "frames: 506000, reward: 1183.000000, loss: 2.499537, episode:  827\n",
      "frames: 507000, reward: 1146.000000, loss: 2.082155, episode:  829\n",
      "frames: 508000, reward: 1141.000000, loss: 4.626939, episode:  830\n",
      "frames: 509000, reward: 1000.000000, loss: 1.647187, episode:  832\n",
      "frames: 510000, reward: 1068.000000, loss: 1.441354, episode:  834\n",
      "frames: 511000, reward: 1137.000000, loss: 3.327841, episode:  835\n",
      "frames: 512000, reward: 938.000000, loss: 1.173476, episode:  837\n",
      "frames: 513000, reward: 943.000000, loss: 2.322801, episode:  839\n",
      "frames: 514000, reward: 952.000000, loss: 1.836107, episode:  840\n",
      "frames: 515000, reward: 927.000000, loss: 4.214849, episode:  842\n",
      "frames: 516000, reward: 987.000000, loss: 7.552518, episode:  844\n",
      "frames: 517000, reward: 807.000000, loss: 1.593094, episode:  846\n",
      "frames: 518000, reward: 825.000000, loss: 3.055772, episode:  848\n",
      "frames: 519000, reward: 819.000000, loss: 1.979237, episode:  850\n",
      "frames: 520000, reward: 755.000000, loss: 1.370597, episode:  851\n",
      "frames: 521000, reward: 821.000000, loss: 9.088443, episode:  853\n",
      "frames: 522000, reward: 760.000000, loss: 1.856422, episode:  855\n",
      "frames: 523000, reward: 748.000000, loss: 7.884341, episode:  857\n",
      "frames: 524000, reward: 737.000000, loss: 1.141546, episode:  859\n",
      "frames: 525000, reward: 760.000000, loss: 1.694837, episode:  860\n",
      "frames: 526000, reward: 777.000000, loss: 1.331747, episode:  862\n",
      "frames: 527000, reward: 752.000000, loss: 1.289052, episode:  863\n",
      "frames: 528000, reward: 949.000000, loss: 1.271940, episode:  865\n",
      "frames: 529000, reward: 1016.000000, loss: 1.253090, episode:  867\n",
      "frames: 530000, reward: 1061.000000, loss: 1.996604, episode:  868\n",
      "frames: 531000, reward: 1129.000000, loss: 1.482491, episode:  870\n",
      "frames: 532000, reward: 1149.000000, loss: 1.649269, episode:  871\n",
      "frames: 533000, reward: 1181.000000, loss: 1.387136, episode:  873\n",
      "frames: 534000, reward: 936.000000, loss: 2.071600, episode:  875\n",
      "frames: 535000, reward: 933.000000, loss: 8.013126, episode:  877\n",
      "frames: 536000, reward: 842.000000, loss: 1.545636, episode:  879\n",
      "frames: 537000, reward: 697.000000, loss: 2.388206, episode:  881\n",
      "frames: 538000, reward: 1054.000000, loss: 3.901189, episode:  882\n",
      "frames: 539000, reward: 1103.000000, loss: 1.029603, episode:  884\n",
      "frames: 540000, reward: 1202.000000, loss: 1.090402, episode:  885\n",
      "frames: 541000, reward: 1244.000000, loss: 1.993271, episode:  887\n",
      "frames: 542000, reward: 1242.000000, loss: 2.781617, episode:  889\n",
      "frames: 543000, reward: 1259.000000, loss: 7.469931, episode:  891\n",
      "frames: 544000, reward: 893.000000, loss: 1.429612, episode:  892\n",
      "frames: 545000, reward: 906.000000, loss: 8.192962, episode:  894\n",
      "frames: 546000, reward: 760.000000, loss: 1.571549, episode:  896\n",
      "frames: 547000, reward: 705.000000, loss: 3.028390, episode:  898\n",
      "frames: 548000, reward: 688.000000, loss: 2.511108, episode:  900\n",
      "frames: 549000, reward: 661.000000, loss: 1.621552, episode:  902\n",
      "frames: 550000, reward: 586.000000, loss: 2.926529, episode:  905\n",
      "frames: 551000, reward: 665.000000, loss: 1.516009, episode:  906\n",
      "frames: 552000, reward: 694.000000, loss: 1.426306, episode:  908\n",
      "frames: 553000, reward: 720.000000, loss: 8.965747, episode:  910\n",
      "frames: 554000, reward: 793.000000, loss: 1.799150, episode:  911\n",
      "frames: 555000, reward: 778.000000, loss: 5.486234, episode:  913\n",
      "frames: 556000, reward: 813.000000, loss: 2.029590, episode:  915\n",
      "frames: 557000, reward: 771.000000, loss: 2.164767, episode:  917\n",
      "frames: 558000, reward: 852.000000, loss: 4.200194, episode:  919\n",
      "frames: 559000, reward: 756.000000, loss: 1.554072, episode:  921\n",
      "frames: 560000, reward: 789.000000, loss: 0.914802, episode:  922\n",
      "frames: 561000, reward: 826.000000, loss: 3.479905, episode:  924\n",
      "frames: 562000, reward: 810.000000, loss: 1.519864, episode:  926\n",
      "frames: 563000, reward: 759.000000, loss: 2.035887, episode:  928\n",
      "frames: 564000, reward: 718.000000, loss: 2.097292, episode:  930\n",
      "frames: 565000, reward: 660.000000, loss: 2.225332, episode:  932\n",
      "frames: 566000, reward: 636.000000, loss: 2.594052, episode:  934\n",
      "frames: 567000, reward: 791.000000, loss: 1.069769, episode:  935\n",
      "frames: 568000, reward: 813.000000, loss: 3.090344, episode:  937\n",
      "frames: 569000, reward: 803.000000, loss: 3.005738, episode:  938\n",
      "frames: 570000, reward: 880.000000, loss: 2.259683, episode:  940\n",
      "frames: 571000, reward: 919.000000, loss: 8.409489, episode:  942\n",
      "frames: 572000, reward: 871.000000, loss: 2.231052, episode:  943\n",
      "frames: 573000, reward: 765.000000, loss: 9.172119, episode:  945\n",
      "frames: 574000, reward: 969.000000, loss: 2.479749, episode:  946\n",
      "frames: 575000, reward: 969.000000, loss: 3.145322, episode:  947\n",
      "frames: 576000, reward: 1032.000000, loss: 26.360390, episode:  949\n",
      "frames: 577000, reward: 964.000000, loss: 2.806682, episode:  950\n",
      "frames: 578000, reward: 1211.000000, loss: 1.952949, episode:  952\n",
      "frames: 579000, reward: 1262.000000, loss: 3.857284, episode:  953\n",
      "frames: 580000, reward: 1322.000000, loss: 1.112915, episode:  954\n",
      "frames: 581000, reward: 1137.000000, loss: 1.859128, episode:  956\n",
      "frames: 582000, reward: 1149.000000, loss: 1.168376, episode:  957\n",
      "frames: 583000, reward: 1142.000000, loss: 2.342535, episode:  959\n",
      "frames: 584000, reward: 1207.000000, loss: 2.693235, episode:  960\n",
      "frames: 585000, reward: 966.000000, loss: 2.391313, episode:  962\n",
      "frames: 586000, reward: 958.000000, loss: 2.428512, episode:  963\n",
      "frames: 587000, reward: 1092.000000, loss: 2.519960, episode:  965\n",
      "frames: 588000, reward: 1076.000000, loss: 1.731660, episode:  966\n",
      "frames: 589000, reward: 1104.000000, loss: 1.363567, episode:  967\n",
      "frames: 590000, reward: 1124.000000, loss: 1.716039, episode:  969\n",
      "frames: 591000, reward: 1077.000000, loss: 1.066013, episode:  970\n",
      "frames: 592000, reward: 1285.000000, loss: 1.846070, episode:  971\n",
      "frames: 593000, reward: 1287.000000, loss: 2.014386, episode:  973\n",
      "frames: 594000, reward: 1094.000000, loss: 4.510462, episode:  975\n",
      "frames: 595000, reward: 1104.000000, loss: 1.966686, episode:  976\n",
      "frames: 596000, reward: 1023.000000, loss: 2.565179, episode:  978\n",
      "frames: 597000, reward: 909.000000, loss: 1.478446, episode:  980\n",
      "frames: 598000, reward: 727.000000, loss: 1.888971, episode:  982\n",
      "frames: 599000, reward: 723.000000, loss: 1.785678, episode:  983\n",
      "frames: 600000, reward: 751.000000, loss: 1.532839, episode:  985\n",
      "frames: 601000, reward: 802.000000, loss: 1.731349, episode:  987\n",
      "frames: 602000, reward: 845.000000, loss: 2.038656, episode:  988\n",
      "frames: 603000, reward: 922.000000, loss: 1.563775, episode:  990\n",
      "frames: 604000, reward: 925.000000, loss: 6.636671, episode:  991\n",
      "frames: 605000, reward: 889.000000, loss: 2.504083, episode:  993\n",
      "frames: 606000, reward: 867.000000, loss: 1.716737, episode:  995\n",
      "frames: 607000, reward: 795.000000, loss: 3.030915, episode:  997\n",
      "frames: 608000, reward: 975.000000, loss: 2.636695, episode:  998\n",
      "frames: 609000, reward: 1076.000000, loss: 1.765243, episode:  999\n",
      "frames: 610000, reward: 1163.000000, loss: 1.389950, episode: 1001\n",
      "frames: 611000, reward: 1180.000000, loss: 1.790807, episode: 1002\n",
      "frames: 612000, reward: 1146.000000, loss: 3.658799, episode: 1004\n",
      "frames: 613000, reward: 1195.000000, loss: 2.189228, episode: 1005\n",
      "frames: 614000, reward: 1246.000000, loss: 2.946907, episode: 1007\n",
      "frames: 615000, reward: 1221.000000, loss: 2.093824, episode: 1008\n",
      "frames: 616000, reward: 1034.000000, loss: 1.564996, episode: 1010\n",
      "frames: 617000, reward: 1049.000000, loss: 9.120440, episode: 1012\n",
      "frames: 618000, reward: 1142.000000, loss: 1.683492, episode: 1014\n",
      "frames: 619000, reward: 1092.000000, loss: 3.242582, episode: 1016\n",
      "frames: 620000, reward: 857.000000, loss: 1.413503, episode: 1018\n",
      "frames: 621000, reward: 874.000000, loss: 7.813316, episode: 1020\n",
      "frames: 622000, reward: 839.000000, loss: 14.434730, episode: 1022\n",
      "frames: 623000, reward: 741.000000, loss: 1.520342, episode: 1023\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 624000, reward: 750.000000, loss: 3.090223, episode: 1025\n",
      "frames: 625000, reward: 758.000000, loss: 7.398223, episode: 1027\n",
      "frames: 626000, reward: 754.000000, loss: 2.053892, episode: 1029\n",
      "frames: 627000, reward: 683.000000, loss: 1.354255, episode: 1031\n",
      "frames: 628000, reward: 610.000000, loss: 1.622201, episode: 1033\n",
      "frames: 629000, reward: 711.000000, loss: 2.103521, episode: 1035\n",
      "frames: 630000, reward: 724.000000, loss: 1.979334, episode: 1037\n",
      "frames: 631000, reward: 741.000000, loss: 7.090187, episode: 1038\n",
      "frames: 632000, reward: 796.000000, loss: 2.060515, episode: 1040\n",
      "frames: 633000, reward: 865.000000, loss: 7.731216, episode: 1041\n",
      "frames: 634000, reward: 879.000000, loss: 1.302138, episode: 1043\n",
      "frames: 635000, reward: 821.000000, loss: 1.679956, episode: 1045\n",
      "frames: 636000, reward: 829.000000, loss: 8.593176, episode: 1046\n",
      "frames: 637000, reward: 1068.000000, loss: 9.906944, episode: 1049\n",
      "frames: 638000, reward: 1082.000000, loss: 8.551301, episode: 1050\n",
      "frames: 639000, reward: 1023.000000, loss: 4.420011, episode: 1052\n",
      "frames: 640000, reward: 1158.000000, loss: 1.657101, episode: 1053\n",
      "frames: 641000, reward: 1202.000000, loss: 1.443651, episode: 1055\n",
      "frames: 642000, reward: 851.000000, loss: 2.370549, episode: 1057\n",
      "frames: 643000, reward: 887.000000, loss: 2.588626, episode: 1058\n",
      "frames: 644000, reward: 1064.000000, loss: 4.149138, episode: 1060\n",
      "frames: 645000, reward: 1065.000000, loss: 2.430910, episode: 1062\n",
      "frames: 646000, reward: 965.000000, loss: 1.040185, episode: 1063\n",
      "frames: 647000, reward: 927.000000, loss: 6.363576, episode: 1064\n",
      "frames: 648000, reward: 1156.000000, loss: 2.115146, episode: 1066\n",
      "frames: 649000, reward: 1386.000000, loss: 2.407310, episode: 1067\n",
      "frames: 650000, reward: 1452.000000, loss: 2.631956, episode: 1068\n",
      "frames: 651000, reward: 1274.000000, loss: 2.836645, episode: 1070\n",
      "frames: 652000, reward: 1344.000000, loss: 2.111143, episode: 1071\n",
      "frames: 653000, reward: 1386.000000, loss: 1.833498, episode: 1072\n",
      "frames: 654000, reward: 1408.000000, loss: 2.810202, episode: 1073\n",
      "frames: 655000, reward: 1417.000000, loss: 7.514732, episode: 1075\n",
      "frames: 656000, reward: 1336.000000, loss: 2.610023, episode: 1076\n",
      "frames: 657000, reward: 1146.000000, loss: 1.679141, episode: 1078\n",
      "frames: 658000, reward: 1289.000000, loss: 2.141525, episode: 1079\n",
      "frames: 659000, reward: 1336.000000, loss: 3.865951, episode: 1081\n",
      "frames: 660000, reward: 1451.000000, loss: 4.655624, episode: 1082\n",
      "frames: 661000, reward: 1481.000000, loss: 12.415611, episode: 1083\n",
      "frames: 662000, reward: 1396.000000, loss: 2.506062, episode: 1085\n",
      "frames: 663000, reward: 1386.000000, loss: 1.470829, episode: 1086\n",
      "frames: 664000, reward: 1450.000000, loss: 2.447408, episode: 1088\n",
      "frames: 665000, reward: 1198.000000, loss: 3.470302, episode: 1090\n",
      "frames: 666000, reward: 1021.000000, loss: 2.022589, episode: 1092\n",
      "frames: 667000, reward: 1012.000000, loss: 2.203880, episode: 1093\n",
      "frames: 668000, reward: 939.000000, loss: 3.295335, episode: 1094\n",
      "frames: 669000, reward: 1063.000000, loss: 1.778603, episode: 1095\n",
      "frames: 670000, reward: 941.000000, loss: 1.526983, episode: 1097\n",
      "frames: 671000, reward: 930.000000, loss: 1.946156, episode: 1098\n",
      "frames: 672000, reward: 1075.000000, loss: 1.096014, episode: 1100\n",
      "frames: 673000, reward: 919.000000, loss: 2.709350, episode: 1103\n",
      "frames: 674000, reward: 1003.000000, loss: 1.654811, episode: 1104\n",
      "frames: 675000, reward: 821.000000, loss: 2.045016, episode: 1106\n",
      "frames: 676000, reward: 1013.000000, loss: 2.126972, episode: 1107\n",
      "frames: 677000, reward: 1170.000000, loss: 2.647906, episode: 1109\n",
      "frames: 678000, reward: 1247.000000, loss: 5.787275, episode: 1110\n",
      "frames: 679000, reward: 1271.000000, loss: 2.883844, episode: 1111\n",
      "frames: 680000, reward: 1352.000000, loss: 1.553355, episode: 1113\n",
      "frames: 681000, reward: 1257.000000, loss: 1.267870, episode: 1115\n",
      "frames: 682000, reward: 1312.000000, loss: 2.285785, episode: 1116\n",
      "frames: 683000, reward: 1112.000000, loss: 2.132163, episode: 1118\n",
      "frames: 684000, reward: 788.000000, loss: 2.113375, episode: 1120\n",
      "frames: 685000, reward: 761.000000, loss: 2.875189, episode: 1122\n",
      "frames: 686000, reward: 831.000000, loss: 1.464836, episode: 1123\n",
      "frames: 687000, reward: 860.000000, loss: 1.294855, episode: 1125\n",
      "frames: 688000, reward: 845.000000, loss: 4.299345, episode: 1127\n",
      "frames: 689000, reward: 861.000000, loss: 2.805829, episode: 1129\n",
      "frames: 690000, reward: 881.000000, loss: 1.269077, episode: 1131\n",
      "frames: 691000, reward: 854.000000, loss: 2.218261, episode: 1132\n",
      "frames: 692000, reward: 843.000000, loss: 2.017920, episode: 1133\n",
      "frames: 693000, reward: 1183.000000, loss: 2.475543, episode: 1135\n",
      "frames: 694000, reward: 1224.000000, loss: 7.408456, episode: 1137\n",
      "frames: 695000, reward: 1215.000000, loss: 1.025738, episode: 1138\n",
      "frames: 696000, reward: 1309.000000, loss: 2.424693, episode: 1139\n",
      "frames: 697000, reward: 1469.000000, loss: 2.273918, episode: 1141\n",
      "frames: 698000, reward: 1557.000000, loss: 2.179762, episode: 1142\n",
      "frames: 699000, reward: 1526.000000, loss: 1.640519, episode: 1143\n",
      "frames: 700000, reward: 1428.000000, loss: 2.915436, episode: 1145\n",
      "frames: 701000, reward: 1291.000000, loss: 8.300200, episode: 1147\n",
      "frames: 702000, reward: 1368.000000, loss: 6.504727, episode: 1148\n",
      "frames: 703000, reward: 1379.000000, loss: 2.039031, episode: 1150\n",
      "frames: 704000, reward: 1410.000000, loss: 1.556136, episode: 1151\n",
      "frames: 705000, reward: 1386.000000, loss: 2.429419, episode: 1153\n",
      "frames: 706000, reward: 1423.000000, loss: 6.075249, episode: 1154\n",
      "frames: 707000, reward: 1471.000000, loss: 2.230208, episode: 1156\n",
      "frames: 708000, reward: 1576.000000, loss: 3.309434, episode: 1157\n",
      "frames: 709000, reward: 1547.000000, loss: 2.960401, episode: 1158\n",
      "frames: 710000, reward: 1387.000000, loss: 8.772139, episode: 1160\n",
      "frames: 711000, reward: 1410.000000, loss: 1.855080, episode: 1161\n",
      "frames: 712000, reward: 1436.000000, loss: 2.408795, episode: 1162\n",
      "frames: 713000, reward: 1286.000000, loss: 8.729316, episode: 1164\n",
      "frames: 714000, reward: 1318.000000, loss: 1.846203, episode: 1165\n",
      "frames: 715000, reward: 1428.000000, loss: 2.258585, episode: 1167\n",
      "frames: 716000, reward: 1428.000000, loss: 2.278254, episode: 1168\n",
      "frames: 717000, reward: 1496.000000, loss: 1.342701, episode: 1170\n",
      "frames: 718000, reward: 1375.000000, loss: 2.251092, episode: 1172\n",
      "frames: 719000, reward: 1437.000000, loss: 2.350933, episode: 1173\n",
      "frames: 720000, reward: 1529.000000, loss: 1.261274, episode: 1174\n",
      "frames: 721000, reward: 1379.000000, loss: 1.821919, episode: 1176\n",
      "frames: 722000, reward: 1353.000000, loss: 1.284717, episode: 1177\n",
      "frames: 723000, reward: 1361.000000, loss: 2.414106, episode: 1178\n",
      "frames: 724000, reward: 1362.000000, loss: 3.315630, episode: 1179\n",
      "frames: 725000, reward: 1336.000000, loss: 1.305724, episode: 1180\n",
      "frames: 726000, reward: 1482.000000, loss: 2.036254, episode: 1182\n",
      "frames: 727000, reward: 1379.000000, loss: 13.289987, episode: 1184\n",
      "frames: 728000, reward: 1315.000000, loss: 1.556613, episode: 1185\n",
      "frames: 729000, reward: 1395.000000, loss: 3.609912, episode: 1186\n",
      "frames: 730000, reward: 1541.000000, loss: 8.412206, episode: 1188\n",
      "frames: 731000, reward: 1487.000000, loss: 2.430836, episode: 1189\n",
      "frames: 732000, reward: 1424.000000, loss: 1.716277, episode: 1191\n",
      "frames: 733000, reward: 1449.000000, loss: 2.007774, episode: 1193\n",
      "frames: 734000, reward: 1385.000000, loss: 5.005645, episode: 1194\n",
      "frames: 735000, reward: 1550.000000, loss: 3.795167, episode: 1195\n",
      "frames: 736000, reward: 1535.000000, loss: 1.897934, episode: 1196\n",
      "frames: 737000, reward: 1636.000000, loss: 1.234349, episode: 1197\n",
      "frames: 738000, reward: 1695.000000, loss: 1.872938, episode: 1198\n",
      "frames: 739000, reward: 1953.000000, loss: 2.703761, episode: 1199\n",
      "frames: 740000, reward: 2175.000000, loss: 5.865460, episode: 1201\n",
      "frames: 741000, reward: 2308.000000, loss: 4.429013, episode: 1203\n",
      "frames: 742000, reward: 2435.000000, loss: 2.466278, episode: 1204\n",
      "frames: 743000, reward: 2297.000000, loss: 1.987587, episode: 1206\n",
      "frames: 744000, reward: 1908.000000, loss: 2.383940, episode: 1208\n",
      "frames: 745000, reward: 1662.000000, loss: 2.891535, episode: 1209\n",
      "frames: 746000, reward: 1326.000000, loss: 2.238455, episode: 1211\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 747000, reward: 1282.000000, loss: 4.005899, episode: 1212\n",
      "frames: 748000, reward: 1185.000000, loss: 2.132748, episode: 1214\n",
      "frames: 749000, reward: 1079.000000, loss: 2.886289, episode: 1216\n",
      "frames: 750000, reward: 1199.000000, loss: 2.312757, episode: 1217\n",
      "frames: 751000, reward: 1296.000000, loss: 3.538574, episode: 1218\n",
      "frames: 752000, reward: 1396.000000, loss: 1.424737, episode: 1220\n",
      "frames: 753000, reward: 1437.000000, loss: 2.501170, episode: 1221\n",
      "frames: 754000, reward: 1420.000000, loss: 3.022464, episode: 1223\n",
      "frames: 755000, reward: 1460.000000, loss: 4.087995, episode: 1224\n",
      "frames: 756000, reward: 1441.000000, loss: 2.399625, episode: 1226\n",
      "frames: 757000, reward: 1201.000000, loss: 2.879582, episode: 1228\n",
      "frames: 758000, reward: 1109.000000, loss: 2.429926, episode: 1230\n",
      "frames: 759000, reward: 1113.000000, loss: 1.908309, episode: 1231\n",
      "frames: 760000, reward: 1071.000000, loss: 2.446694, episode: 1233\n",
      "frames: 761000, reward: 1094.000000, loss: 2.153900, episode: 1234\n",
      "frames: 762000, reward: 1121.000000, loss: 1.793297, episode: 1235\n",
      "frames: 763000, reward: 1281.000000, loss: 2.135463, episode: 1237\n",
      "frames: 764000, reward: 1261.000000, loss: 4.256142, episode: 1238\n",
      "frames: 765000, reward: 1149.000000, loss: 2.334423, episode: 1239\n",
      "frames: 766000, reward: 1284.000000, loss: 2.223563, episode: 1241\n",
      "frames: 767000, reward: 1329.000000, loss: 2.671160, episode: 1242\n",
      "frames: 768000, reward: 1569.000000, loss: 9.474403, episode: 1243\n",
      "frames: 769000, reward: 1720.000000, loss: 2.854059, episode: 1245\n",
      "frames: 770000, reward: 1687.000000, loss: 3.623690, episode: 1246\n",
      "frames: 771000, reward: 1544.000000, loss: 2.406204, episode: 1248\n",
      "frames: 772000, reward: 1582.000000, loss: 2.350152, episode: 1250\n",
      "frames: 773000, reward: 1654.000000, loss: 7.762800, episode: 1251\n",
      "frames: 774000, reward: 1633.000000, loss: 2.697633, episode: 1252\n",
      "frames: 775000, reward: 1307.000000, loss: 2.072852, episode: 1254\n",
      "frames: 776000, reward: 1089.000000, loss: 2.030641, episode: 1256\n",
      "frames: 777000, reward: 1153.000000, loss: 2.153267, episode: 1258\n",
      "frames: 778000, reward: 1155.000000, loss: 9.930779, episode: 1259\n",
      "frames: 779000, reward: 1131.000000, loss: 2.078062, episode: 1260\n",
      "frames: 780000, reward: 1019.000000, loss: 2.891831, episode: 1261\n",
      "frames: 781000, reward: 1031.000000, loss: 2.608462, episode: 1262\n",
      "frames: 782000, reward: 1075.000000, loss: 3.327118, episode: 1264\n",
      "frames: 783000, reward: 1229.000000, loss: 1.856516, episode: 1265\n",
      "frames: 784000, reward: 1332.000000, loss: 3.075540, episode: 1267\n",
      "frames: 785000, reward: 1413.000000, loss: 2.137022, episode: 1268\n",
      "frames: 786000, reward: 1491.000000, loss: 2.358617, episode: 1269\n",
      "frames: 787000, reward: 1525.000000, loss: 1.479075, episode: 1270\n",
      "frames: 788000, reward: 1555.000000, loss: 1.966769, episode: 1272\n",
      "frames: 789000, reward: 1499.000000, loss: 2.326205, episode: 1273\n",
      "frames: 790000, reward: 1556.000000, loss: 2.278581, episode: 1274\n",
      "frames: 791000, reward: 1572.000000, loss: 2.384970, episode: 1276\n",
      "frames: 792000, reward: 1717.000000, loss: 3.331376, episode: 1277\n",
      "frames: 793000, reward: 1674.000000, loss: 2.697526, episode: 1278\n",
      "frames: 794000, reward: 1661.000000, loss: 1.734778, episode: 1280\n",
      "frames: 795000, reward: 1705.000000, loss: 2.191590, episode: 1281\n",
      "frames: 796000, reward: 1536.000000, loss: 2.478117, episode: 1283\n",
      "frames: 797000, reward: 1808.000000, loss: 1.848422, episode: 1284\n",
      "frames: 798000, reward: 1805.000000, loss: 2.206714, episode: 1285\n",
      "frames: 799000, reward: 1690.000000, loss: 1.910953, episode: 1286\n",
      "frames: 800000, reward: 1604.000000, loss: 1.641764, episode: 1287\n",
      "frames: 801000, reward: 1613.000000, loss: 3.098431, episode: 1289\n",
      "frames: 802000, reward: 1552.000000, loss: 9.699656, episode: 1290\n",
      "frames: 803000, reward: 1632.000000, loss: 3.232606, episode: 1292\n",
      "frames: 804000, reward: 1695.000000, loss: 1.391410, episode: 1293\n",
      "frames: 805000, reward: 1754.000000, loss: 3.161096, episode: 1295\n",
      "frames: 806000, reward: 1904.000000, loss: 1.262511, episode: 1296\n",
      "frames: 807000, reward: 1759.000000, loss: 2.086018, episode: 1298\n",
      "frames: 808000, reward: 1767.000000, loss: 3.101658, episode: 1300\n",
      "frames: 809000, reward: 1721.000000, loss: 2.736463, episode: 1301\n",
      "frames: 810000, reward: 1721.000000, loss: 2.275097, episode: 1303\n",
      "frames: 811000, reward: 1796.000000, loss: 2.018251, episode: 1304\n",
      "frames: 812000, reward: 1520.000000, loss: 2.779146, episode: 1305\n",
      "frames: 813000, reward: 1460.000000, loss: 1.933172, episode: 1307\n",
      "frames: 814000, reward: 1458.000000, loss: 8.035908, episode: 1308\n",
      "frames: 815000, reward: 1665.000000, loss: 2.813561, episode: 1309\n",
      "frames: 816000, reward: 1732.000000, loss: 2.516164, episode: 1310\n",
      "frames: 817000, reward: 1832.000000, loss: 2.369738, episode: 1311\n",
      "frames: 818000, reward: 1868.000000, loss: 2.756041, episode: 1312\n",
      "frames: 819000, reward: 1729.000000, loss: 2.522926, episode: 1314\n",
      "frames: 820000, reward: 1703.000000, loss: 1.645560, episode: 1315\n",
      "frames: 821000, reward: 1722.000000, loss: 1.643373, episode: 1317\n",
      "frames: 822000, reward: 1749.000000, loss: 2.748609, episode: 1318\n",
      "frames: 823000, reward: 1682.000000, loss: 2.495472, episode: 1319\n",
      "frames: 824000, reward: 1692.000000, loss: 1.655771, episode: 1320\n",
      "frames: 825000, reward: 1795.000000, loss: 1.558093, episode: 1321\n",
      "frames: 826000, reward: 1774.000000, loss: 3.050561, episode: 1322\n",
      "frames: 827000, reward: 1846.000000, loss: 12.918203, episode: 1323\n",
      "frames: 828000, reward: 2177.000000, loss: 2.058891, episode: 1325\n",
      "frames: 829000, reward: 2171.000000, loss: 2.166644, episode: 1326\n",
      "frames: 830000, reward: 2159.000000, loss: 2.322594, episode: 1328\n",
      "frames: 831000, reward: 2047.000000, loss: 1.902881, episode: 1329\n",
      "frames: 832000, reward: 2288.000000, loss: 5.622162, episode: 1330\n",
      "frames: 833000, reward: 2287.000000, loss: 1.431213, episode: 1331\n",
      "frames: 834000, reward: 2312.000000, loss: 1.879018, episode: 1332\n",
      "frames: 835000, reward: 2498.000000, loss: 1.454791, episode: 1334\n",
      "frames: 836000, reward: 2469.000000, loss: 2.639684, episode: 1335\n",
      "frames: 837000, reward: 2461.000000, loss: 1.723701, episode: 1336\n",
      "frames: 838000, reward: 2465.000000, loss: 2.757260, episode: 1338\n",
      "frames: 839000, reward: 2225.000000, loss: 1.974552, episode: 1340\n",
      "frames: 840000, reward: 2225.000000, loss: 2.425591, episode: 1340\n",
      "frames: 841000, reward: 2145.000000, loss: 3.733596, episode: 1342\n",
      "frames: 842000, reward: 1827.000000, loss: 1.787411, episode: 1343\n",
      "frames: 843000, reward: 1551.000000, loss: 7.627605, episode: 1345\n",
      "frames: 844000, reward: 1541.000000, loss: 2.144423, episode: 1346\n",
      "frames: 845000, reward: 1463.000000, loss: 2.857850, episode: 1348\n",
      "frames: 846000, reward: 1369.000000, loss: 7.470667, episode: 1349\n",
      "frames: 847000, reward: 1457.000000, loss: 1.865413, episode: 1350\n",
      "frames: 848000, reward: 1465.000000, loss: 1.966497, episode: 1352\n",
      "frames: 849000, reward: 1469.000000, loss: 2.093455, episode: 1353\n",
      "frames: 850000, reward: 1501.000000, loss: 3.104436, episode: 1354\n",
      "frames: 851000, reward: 1564.000000, loss: 9.797500, episode: 1356\n",
      "frames: 852000, reward: 1611.000000, loss: 6.144001, episode: 1358\n",
      "frames: 853000, reward: 1615.000000, loss: 19.959814, episode: 1359\n",
      "frames: 854000, reward: 1472.000000, loss: 1.486283, episode: 1361\n",
      "frames: 855000, reward: 1627.000000, loss: 2.861030, episode: 1362\n",
      "frames: 856000, reward: 1515.000000, loss: 1.821928, episode: 1364\n",
      "frames: 857000, reward: 1466.000000, loss: 10.953950, episode: 1365\n",
      "frames: 858000, reward: 1509.000000, loss: 1.943395, episode: 1367\n",
      "frames: 859000, reward: 1572.000000, loss: 3.971193, episode: 1369\n",
      "frames: 860000, reward: 1503.000000, loss: 2.197756, episode: 1370\n",
      "frames: 861000, reward: 1617.000000, loss: 1.789071, episode: 1371\n",
      "frames: 862000, reward: 1454.000000, loss: 2.883716, episode: 1372\n",
      "frames: 863000, reward: 1659.000000, loss: 2.961912, episode: 1374\n",
      "frames: 864000, reward: 1718.000000, loss: 2.319477, episode: 1375\n",
      "frames: 865000, reward: 1704.000000, loss: 1.151643, episode: 1377\n",
      "frames: 866000, reward: 1719.000000, loss: 11.464614, episode: 1378\n",
      "frames: 867000, reward: 1694.000000, loss: 2.454329, episode: 1379\n",
      "frames: 868000, reward: 1672.000000, loss: 3.005781, episode: 1381\n",
      "frames: 869000, reward: 1636.000000, loss: 3.391876, episode: 1382\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 870000, reward: 1516.000000, loss: 1.287262, episode: 1384\n",
      "frames: 871000, reward: 1434.000000, loss: 4.168095, episode: 1385\n",
      "frames: 872000, reward: 1468.000000, loss: 1.612667, episode: 1387\n",
      "frames: 873000, reward: 1559.000000, loss: 1.761851, episode: 1388\n",
      "frames: 874000, reward: 1734.000000, loss: 2.541941, episode: 1389\n",
      "frames: 875000, reward: 1710.000000, loss: 4.901145, episode: 1390\n",
      "frames: 876000, reward: 1668.000000, loss: 2.093805, episode: 1392\n",
      "frames: 877000, reward: 1636.000000, loss: 2.118916, episode: 1394\n",
      "frames: 878000, reward: 1657.000000, loss: 3.292109, episode: 1396\n",
      "frames: 879000, reward: 1621.000000, loss: 2.207562, episode: 1397\n",
      "frames: 880000, reward: 1657.000000, loss: 2.067880, episode: 1398\n",
      "frames: 881000, reward: 1595.000000, loss: 1.980395, episode: 1400\n",
      "frames: 882000, reward: 1655.000000, loss: 1.966318, episode: 1401\n",
      "frames: 883000, reward: 1857.000000, loss: 2.705209, episode: 1403\n",
      "frames: 884000, reward: 1907.000000, loss: 1.996612, episode: 1404\n",
      "frames: 885000, reward: 2037.000000, loss: 2.645552, episode: 1405\n",
      "frames: 886000, reward: 2072.000000, loss: 2.360793, episode: 1407\n",
      "frames: 887000, reward: 2039.000000, loss: 1.305869, episode: 1408\n",
      "frames: 888000, reward: 2037.000000, loss: 1.585742, episode: 1410\n",
      "frames: 889000, reward: 1988.000000, loss: 1.396609, episode: 1411\n",
      "frames: 890000, reward: 2073.000000, loss: 2.195348, episode: 1412\n",
      "frames: 891000, reward: 1879.000000, loss: 6.704518, episode: 1414\n",
      "frames: 892000, reward: 1753.000000, loss: 2.112397, episode: 1415\n",
      "frames: 893000, reward: 1742.000000, loss: 12.361908, episode: 1416\n",
      "frames: 894000, reward: 1742.000000, loss: 2.480047, episode: 1418\n",
      "frames: 895000, reward: 1863.000000, loss: 2.470936, episode: 1419\n",
      "frames: 896000, reward: 1927.000000, loss: 2.077700, episode: 1420\n",
      "frames: 897000, reward: 1918.000000, loss: 3.082038, episode: 1422\n",
      "frames: 898000, reward: 1828.000000, loss: 13.679176, episode: 1423\n",
      "frames: 899000, reward: 1847.000000, loss: 1.854137, episode: 1424\n",
      "frames: 900000, reward: 1861.000000, loss: 3.998150, episode: 1426\n",
      "frames: 901000, reward: 1865.000000, loss: 2.094299, episode: 1427\n",
      "frames: 902000, reward: 1954.000000, loss: 1.941541, episode: 1428\n",
      "frames: 903000, reward: 1638.000000, loss: 2.251218, episode: 1430\n",
      "frames: 904000, reward: 1837.000000, loss: 1.487100, episode: 1431\n",
      "frames: 905000, reward: 1892.000000, loss: 2.030172, episode: 1432\n",
      "frames: 906000, reward: 2021.000000, loss: 2.664051, episode: 1433\n",
      "frames: 907000, reward: 2257.000000, loss: 6.603855, episode: 1435\n",
      "frames: 908000, reward: 2255.000000, loss: 10.347038, episode: 1436\n",
      "frames: 909000, reward: 2151.000000, loss: 3.785183, episode: 1437\n",
      "frames: 910000, reward: 2074.000000, loss: 10.609961, episode: 1439\n",
      "frames: 911000, reward: 2060.000000, loss: 1.448683, episode: 1440\n",
      "frames: 912000, reward: 1806.000000, loss: 4.160136, episode: 1442\n",
      "frames: 913000, reward: 1704.000000, loss: 1.699087, episode: 1443\n",
      "frames: 914000, reward: 1651.000000, loss: 3.366037, episode: 1444\n",
      "frames: 915000, reward: 1511.000000, loss: 2.382689, episode: 1446\n",
      "frames: 916000, reward: 1459.000000, loss: 0.923263, episode: 1448\n",
      "frames: 917000, reward: 1438.000000, loss: 26.821033, episode: 1450\n",
      "frames: 918000, reward: 1527.000000, loss: 2.055637, episode: 1451\n",
      "frames: 919000, reward: 1556.000000, loss: 8.912931, episode: 1452\n",
      "frames: 920000, reward: 1537.000000, loss: 13.549923, episode: 1454\n",
      "frames: 921000, reward: 1600.000000, loss: 2.732822, episode: 1456\n",
      "frames: 922000, reward: 1574.000000, loss: 5.913362, episode: 1457\n",
      "frames: 923000, reward: 1605.000000, loss: 2.686505, episode: 1458\n",
      "frames: 924000, reward: 1869.000000, loss: 3.955135, episode: 1460\n",
      "frames: 925000, reward: 1644.000000, loss: 7.885833, episode: 1462\n",
      "frames: 926000, reward: 1720.000000, loss: 2.695652, episode: 1463\n",
      "frames: 927000, reward: 1748.000000, loss: 3.284242, episode: 1465\n",
      "frames: 928000, reward: 1747.000000, loss: 2.742712, episode: 1466\n",
      "frames: 929000, reward: 1861.000000, loss: 1.802789, episode: 1468\n",
      "frames: 930000, reward: 1738.000000, loss: 1.686756, episode: 1469\n",
      "frames: 931000, reward: 1723.000000, loss: 2.566638, episode: 1470\n",
      "frames: 932000, reward: 1810.000000, loss: 2.686799, episode: 1471\n",
      "frames: 933000, reward: 2007.000000, loss: 2.217877, episode: 1472\n",
      "frames: 934000, reward: 1913.000000, loss: 2.236436, episode: 1473\n",
      "frames: 935000, reward: 2030.000000, loss: 2.025348, episode: 1475\n",
      "frames: 936000, reward: 2087.000000, loss: 5.655408, episode: 1476\n",
      "frames: 937000, reward: 1996.000000, loss: 11.458145, episode: 1478\n",
      "frames: 938000, reward: 2032.000000, loss: 4.740733, episode: 1479\n",
      "frames: 939000, reward: 1834.000000, loss: 2.380318, episode: 1481\n",
      "frames: 940000, reward: 1785.000000, loss: 2.038628, episode: 1482\n",
      "frames: 941000, reward: 1649.000000, loss: 3.637761, episode: 1484\n",
      "frames: 942000, reward: 1692.000000, loss: 4.155735, episode: 1485\n",
      "frames: 943000, reward: 1652.000000, loss: 3.117034, episode: 1486\n",
      "frames: 944000, reward: 1788.000000, loss: 2.343915, episode: 1487\n",
      "frames: 945000, reward: 1983.000000, loss: 3.951874, episode: 1488\n",
      "frames: 946000, reward: 2032.000000, loss: 2.374172, episode: 1490\n",
      "frames: 947000, reward: 2073.000000, loss: 2.594626, episode: 1491\n",
      "frames: 948000, reward: 2006.000000, loss: 4.709947, episode: 1492\n",
      "frames: 949000, reward: 1947.000000, loss: 1.797206, episode: 1494\n",
      "frames: 950000, reward: 1955.000000, loss: 1.534697, episode: 1495\n",
      "frames: 951000, reward: 1793.000000, loss: 4.103490, episode: 1496\n",
      "frames: 952000, reward: 1763.000000, loss: 7.622529, episode: 1497\n",
      "frames: 953000, reward: 1633.000000, loss: 1.793828, episode: 1499\n",
      "frames: 954000, reward: 1712.000000, loss: 2.917720, episode: 1500\n",
      "frames: 955000, reward: 1746.000000, loss: 2.111475, episode: 1502\n",
      "frames: 956000, reward: 1905.000000, loss: 2.067048, episode: 1503\n",
      "frames: 957000, reward: 2002.000000, loss: 2.311593, episode: 1504\n",
      "frames: 958000, reward: 2011.000000, loss: 1.420356, episode: 1506\n",
      "frames: 959000, reward: 1863.000000, loss: 2.570173, episode: 1507\n",
      "frames: 960000, reward: 1827.000000, loss: 2.106689, episode: 1508\n",
      "frames: 961000, reward: 1827.000000, loss: 2.752109, episode: 1508\n",
      "frames: 962000, reward: 1733.000000, loss: 2.514250, episode: 1510\n",
      "frames: 963000, reward: 1608.000000, loss: 2.538011, episode: 1511\n",
      "frames: 964000, reward: 1471.000000, loss: 2.134270, episode: 1513\n",
      "frames: 965000, reward: 1316.000000, loss: 2.514877, episode: 1515\n",
      "frames: 966000, reward: 1564.000000, loss: 1.954780, episode: 1516\n",
      "frames: 967000, reward: 1584.000000, loss: 2.220724, episode: 1517\n",
      "frames: 968000, reward: 1396.000000, loss: 2.054486, episode: 1519\n",
      "frames: 969000, reward: 1459.000000, loss: 3.081961, episode: 1520\n",
      "frames: 970000, reward: 1420.000000, loss: 3.344464, episode: 1521\n",
      "frames: 971000, reward: 1504.000000, loss: 6.116940, episode: 1523\n",
      "frames: 972000, reward: 1542.000000, loss: 2.165296, episode: 1524\n",
      "frames: 973000, reward: 1411.000000, loss: 4.086742, episode: 1526\n",
      "frames: 974000, reward: 1362.000000, loss: 15.213484, episode: 1528\n",
      "frames: 975000, reward: 1361.000000, loss: 2.766040, episode: 1529\n",
      "frames: 976000, reward: 1339.000000, loss: 2.158053, episode: 1530\n",
      "frames: 977000, reward: 1292.000000, loss: 2.513752, episode: 1532\n",
      "frames: 978000, reward: 1315.000000, loss: 27.176600, episode: 1534\n",
      "frames: 979000, reward: 1319.000000, loss: 1.804193, episode: 1535\n",
      "frames: 980000, reward: 1384.000000, loss: 1.734564, episode: 1537\n",
      "frames: 981000, reward: 1310.000000, loss: 2.189431, episode: 1538\n",
      "frames: 982000, reward: 1490.000000, loss: 2.603606, episode: 1540\n",
      "frames: 983000, reward: 1433.000000, loss: 3.189172, episode: 1541\n",
      "frames: 984000, reward: 1357.000000, loss: 2.859514, episode: 1543\n",
      "frames: 985000, reward: 1388.000000, loss: 2.097153, episode: 1544\n",
      "frames: 986000, reward: 1442.000000, loss: 13.075150, episode: 1545\n",
      "frames: 987000, reward: 1642.000000, loss: 11.953230, episode: 1546\n",
      "frames: 988000, reward: 1589.000000, loss: 2.246141, episode: 1547\n",
      "frames: 989000, reward: 1751.000000, loss: 2.706724, episode: 1549\n",
      "frames: 990000, reward: 1831.000000, loss: 2.945413, episode: 1550\n",
      "frames: 991000, reward: 1825.000000, loss: 2.433567, episode: 1552\n",
      "frames: 992000, reward: 1768.000000, loss: 2.087621, episode: 1553\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "frames: 993000, reward: 1722.000000, loss: 2.253877, episode: 1555\n",
      "frames: 994000, reward: 1585.000000, loss: 3.291570, episode: 1556\n",
      "frames: 995000, reward: 1495.000000, loss: 2.949622, episode: 1558\n",
      "frames: 996000, reward: 1443.000000, loss: 1.914942, episode: 1559\n",
      "frames: 997000, reward: 1378.000000, loss: 2.403353, episode: 1560\n",
      "frames: 998000, reward: 1700.000000, loss: 2.955715, episode: 1562\n",
      "frames: 999000, reward: 1816.000000, loss: 1.928494, episode: 1563\n"
     ]
    }
   ],
   "source": [
    "# if __name__ == '__main__':\n",
    "    \n",
    "# Training DQN in PongNoFrameskip-v4 \n",
    "env = make_atari('MsPacmanNoFrameskip-v4')\n",
    "env = wrap_deepmind(env, scale = False, frame_stack=True)\n",
    "gamma = 0.99\n",
    "frames = 1000000\n",
    "USE_CUDA = True\n",
    "learning_rate = 2e-4\n",
    "max_buff = 100000\n",
    "update_tar_interval = 1000\n",
    "batch_size = 32\n",
    "print_interval = 1000\n",
    "log_interval = 1000\n",
    "learning_start = 10000\n",
    "update_current_step =4 # update current model every 4 steps\n",
    "\n",
    "action_space = env.action_space\n",
    "action_dim = env.action_space.n\n",
    "state_dim = env.observation_space.shape[0]\n",
    "state_channel = env.observation_space.shape[2]\n",
    "reward_scale = 0.5/action_dim\n",
    "agent = soft_DQNAgent(in_channels = state_channel, action_space= action_space, USE_CUDA = USE_CUDA, lr = learning_rate, memory_size = max_buff, reward_scale = reward_scale)\n",
    "\n",
    "frame = env.reset()\n",
    "\n",
    "episode_reward = 0\n",
    "all_rewards = []\n",
    "losses = []\n",
    "episode_num = 0\n",
    "is_win = False\n",
    "# tensorboard\n",
    "summary_writer = SummaryWriter(log_dir = \"5_Soft_DQN_MsPacman2\", comment= \"good_makeatari\")\n",
    "\n",
    "for i in range(frames):\n",
    "    state_tensor = agent.observe(frame)\n",
    "    action = agent.act(state_tensor, i, learning_start)\n",
    "    \n",
    "    next_frame, reward, is_done, _ = env.step(action)\n",
    "    \n",
    "    episode_reward += reward\n",
    "    agent.memory_buffer.push(frame, action, reward, next_frame, is_done)\n",
    "    frame = next_frame\n",
    "    \n",
    "    loss = 0\n",
    "    if agent.memory_buffer.size() >= learning_start:\n",
    "        if i % update_current_step == 0:\n",
    "            loss = agent.learn_from_experience(batch_size)\n",
    "            losses.append(loss)\n",
    "\n",
    "    if i % print_interval == 0:\n",
    "        print(\"frames: %5d, reward: %5f, loss: %4f, episode: %4d\" % (i, np.mean(all_rewards[-10:]), loss, episode_num))\n",
    "        summary_writer.add_scalar(\"Temporal Difference Loss\", loss, i)\n",
    "        summary_writer.add_scalar(\"Mean Reward\", np.mean(all_rewards[-10:]), i)\n",
    "        \n",
    "    if i % update_tar_interval == 0:\n",
    "        agent.DQN_target.load_state_dict(agent.DQN.state_dict())\n",
    "    \n",
    "    if is_done:\n",
    "        \n",
    "        frame = env.reset()\n",
    "        \n",
    "        all_rewards.append(episode_reward)\n",
    "        episode_reward = 0\n",
    "        episode_num += 1\n",
    "        avg_reward = float(np.mean(all_rewards[-100:]))\n",
    "\n",
    "summary_writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwcAAAE/CAYAAAAT7r+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeZwcZZ0/8M+XhEvkJiCXhhUEwRXQiCgrCiiC6ILrsewq8FNcVhcVb8MlyhkQBBFBkSDhDGe4Qu77Tib3MTkmk8nMJJOZSea+p6ef3x9VPVPTXdV1dJ3dn/frlVem63y6urvq+T6nKKVARERERES0X9QJICIiIiKieGBwQEREREREABgcEBERERGRjsEBEREREREBYHBAREREREQ6BgdERERERASAwUHiicjpIrJaRNpF5KdRp4eiJSJzReQHUaeDiKiYiEiViHwx6nQQhYHBQfL9BsBcpdShSqlHo05MNhH5mohsEJEOEVksImca1h0oIg+LyG4RaRaRx0Vkf8P6j4rIbBFpFZEKEfl61rF/oC/vEJGpInKCYd0RIjJBRBr0f7/P2vezIrJcD6rWici/BXgZYklEPiYi00Rkr4jkTHiS7/qLyHf065751yUiSkQ+qa+fkrW+T0TW50nLJSKyWT/OHBH5UDDvmoiIiPJhcJB8HwKw0WqliIwIMS3Z5z4NwAsAfgjgCADvAHhbREbqm4wFMAbAxwB8BMAnANym7zsSwFsA3gVwFIAbADwvIh/R138ewL0ArtTX7wDwkuH0DwN4H4DRAM4DcI2IfE/f9ygAbwP4o56uBwC8IyJHeniPIiKh/44M17AQ/QBeAXC9xfEtr79S6gWl1Psz/wD8H4BKAKv09ZdnrV8M4FWL93IMgDcA3K6fqwzAyz68PyIiInKJwUGCichsABcBeEwvnf2IiDwjIk+IyHsi0gngIhG5Qm961CYiNcZSdBEZrZf4fk9f1ywiPxSRT+kl6i0i8ljWeb8vIuX6ttPylPJ+GcACpdRCpVQKwP0ATgTweX391wA8qpRqUko1AngUwPf1dWcAOAHAw0qpAaXUbACLAFxj2PdVpdRGpVQfgLsAXCgiHzasf0Ap1aWUqgIw3nDszwKoV0q9qh/7eQCNAP7D4XWfKyL3iMgiAF0A/kVEDheR8SJSJyK7ROTuTGAmIjsNJerf1a/3mfrrH4jIm/rf54nIEv2a14nIYyJygOG8SkRuFJFtALbpy76kl7i36p+TOHkPAKCU2qKUGg/z4NLu+me7DsCzymTKdREZDeBzAJ6z2Pc/AGzUP48eAL8HcLaInOH0vRARhUG0Gu9HRKvx3q3/faC+7hgReVe/hzeJyIJM4ZGI/FZ/NrSLyBYRuSTad0JkjcFBgimlLgawAMCP9RLarfqq/wZwD4BDASwE0AngWmil5FcA+JGIXJV1uE8DOA3AfwJ4BMCtAL4I4CwA39ZL6qHvdwu0DN0o/fwvwZxgeGY18/pjedafJCKHwzyTa7cvDOthst5q3+z1TlwDrTT9UAA7AUwAkAJwKoBzAVwKINP2fx6AL+h/XwithP3zhtfz9L8HAPwcwDEAPgPgEmgl8kZXQfusztRL3F+HVttyDIDtAC4YfEMiH9QfUh908b4Gd7dYlnON9ODwQgDPWhzrWmhB4g6L9WcBWJt5oZTqhPZeznKTYCKiENwK4HwA5wA4G1rN9G36ul8CqIX2bDwO2rNSicjpAH4M4FNKqUOhFZxVhZtsIucYHBSnt5RSi5RSaaVUj1JqrlJqvf56HbTM/Oez9rlL33Y6tGDiJaVUg1JqF7QA4Fx9u/8FcJ9SqlyvDbgXwDkWtQczAHxeRL6gl4DfAuAAaM19AGAKgJtEZJSIfABApkP1+wBsBtAA4Ncisr+IXKqnObPve9CClo+LyMEAfgdAGdZPBTBWRA4VkVOh1Rpk1i0GcIKI/Jd+7OsAfNiw3oln9FqLFLSmMJcD+JlSqlMp1QCtWdPV+rbzMHS9PwfgPsPrz+vroZRaqZRaqpRK6bUdf0fu53SfXtPSDeArADYppV5TSvVDC+r2ZDZUSlUrpY5QSlW7eF8ZdtffyC7zfy2AZ/Kc6/0AWrOWtUILvIiI4uQ7AO7Un4+NAP6AoRrVfgDHA/iQUqpfKbVAr00dAHAgtEKd/ZVSVUqp7ZGknsgBBgfFqcb4QkQ+rXfybBSRVmh9AI7J2qfe8He3yev3639/CMCf9RLpFgBN0EqUT8xOhFJqM7TmJo8BqNPPuQlayQqg1W6sBrAGWob9TWg31wY9s3sVtJqOPdBKZF7J7KuUmgXgDmgl5zuhlcK0G479Uz3d26C1nX/JsO8+aH0VfqG/z8sAzDTs64TxGn8IwP4A6gzX5e8AjtXXzwPwOT0AGgGtPf0FenObw/X3D71Z2LsiskdE2qAFXtmfk/G8Jxhf6w+hGvjA7vpnuRZazUkO0Tp6fwDAa3lO1wHgsKxlh0H7PImI4uQEaM+cjJ36MkDrx1YBYLqIVIrIWABQSlUA+Bm0JpMNIjJRDANoEMUNg4PilN3u+0VoHXBPVkodDuBvcNE2PUsNgP/VS6Qz/w5WSi02TYhWqv0xpdTR0DLzHwKwQl/XrZT6sVLqRKXUvwDYB2ClUmpAX79OKfV5pdTRSqkvA/gXAMsNx/6rUuo0pdSx0IKEkQA26OualFLfUUp9QCl1FrTvunHfeUqpTymljoJW6nO6cb0DxmtcA6AXwDGGa3KYft7Mg6ELWsAyXynVDi3DfQOAhUqptH6cJ6CV2J+mlDoMWk1L9udkPG8dgJMzL0REjK8LZXf99XNeAO3BaJX5vw7AG0qpjjyn2gitej5zzEOg1eRYdrQnIorIbmjPsYwP6suglGpXSv1Sf559DcAvMn0LlFIvKqX+Td9XQeuDRxRLDA5Kw6EAmpRSPSJyHrQ+CV79DcDNInIWAOgdcb9ltbGIfFJERojIKGil6e/oNQoQkRNF5ATRnA9ttJo7DPt+XEQOEpH3icivoFXXPqOvO0i0oThFb1P/JIA/K6Wa9fUfFpGj9XNfDi0jfrfh2OfqzWUOA/AggFql1DQvF0QpVQdgOoCHROQwEdlPP7+xSdA8aG1OM/0L5ma9BrTPqQ1Ah94Z90c2p54M4CwR+Q/RRhf6KbRSekf0a3cQtKZemWt6oGG95fU3uA7A63rAk338gwF8y2SfbJMAfExEvqGn53cA1mW+J0REMfISgNv05rDHQLtfPQ8AIvJVETlVL6hpg9acaEC0+Ygu1u+vPdBqtQciSj+RLQYHpeH/ANwpIu3QbmSveD2QUmoStBKPiXrTlw3Q2ttb+TOAFgBb9P//x7Duw9CaE3VCa5YyVu/zkHENtNLxBmidc7+klOrV1x0ErUakA1pp9hJowUXGJwGsh9Y05T4A31FKGUuifwNgL7RS/+MBGMfw/5yI5CvpNnMttEz2JgDN0ErSjzesnwct8z/f4jUA/Apa4NYO4B+wGc5TKbUXWuZ7HLRal9OgjSiUeR8fFG0UK6sOyR+C9pDKXJduaJ9TRr7rDz0j/21YNCmC1iypFcCc7BUislFEvqO/j0YA34DWzKwZWofrq7P3ISKKgbuhDbe8DtozZhWGCp5Og9ZEtQPaM+lxpdRcaP0NxkF75uyB1uT0llBTTeSCmIw8SEREREREJYg1B0REREREBIDBARERERER6RgcEBERERERAAYHRERERESkY3BAREREREQAtEmjYuuYY45Ro0ePjjoZRESxtHLlyr1KqVFRpyNKfE4QEZnz+oyIdXAwevRolJWVRZ0MIqJYEpGdUachanxOEBGZ8/qMYLMiIiIiIiICwOCAiIiIiIh0DA6IiCh0InKEiLwmIptFpFxEPiMiR4nIDBHZpv9/ZNTpJCIqNQwOiIgoCn8GMFUpdQaAswGUAxgLYJZS6jQAs/TXREQUIgYHREQUKhE5DMCFAMYDgFKqTynVAuBKABP0zSYAuCqaFBIRlS4GB0REFLZ/AdAI4J8islpEnhKRQwAcp5SqAwD9/2PNdhaRG0SkTETKGhsbw0s1EVEJYHBARERhGwngEwCeUEqdC6ATLpoQKaWeVEqNUUqNGTWqpKd5ICLyHYMDIiIKWy2AWqXUMv31a9CChXoROR4A9P8bIkofEVHJYnBAREShUkrtAVAjIqfriy4BsAnA2wCu05ddB+CtCJJHRFTSYj1DMhGRU72pAaza2YLPfPjoqJNCzvwEwAsicgCASgDfg1Zg9YqIXA+gGsC3IkwfEdGgrfXtOPSgkTj+8IOjTkrgGBwQUVG4Z3I5nl2yE+/99HM484TDok4O2VBKrQEwxmTVJWGnhYjIzqUPzwcAVI27IuKUBI/NioioKGytbwcAtHT3RZwSIiKi5GJwQETFRUWdACIiouRicEBEREREkXtqQSXW1LREnYySx+CAiIiIiELVl0rjgamb0dmbGlx29+RyXPXXRRGmKh6q9nbignGzUd/WE8n5GRwQUVEQSNRJICIihyauqMbjc7fjL7Mrok5K7ExYUoVdLd14d11dJOdncEAUsp7+AWzY1Rp1MooWuxwQEcVfXyo97H+KDwYHRCG7ZdJ6fPUvC7GnNZrqQiIiIiIrDA6IQrZW72zV0dsfcUqIiIiIhmNwQERFQdjlgIgocfoH0mjuDGZ+mr0dvbjvvXIMpNng1A0GB0QRUbxX+YrXk4goeZ5buhPn3jUjkGPfNmkD/j6/EvO3NgZy/GI1MuoEEJUaYRE3ERFR4PoGtM7O6QJKjy56cC4+ftLhfiUpERgcEIVMsYg7EIy5iIjIbzv2dmLH3s6okxEqNisiiggzs8Fg7EVE5I/H51bg2SVVUSeDQsbggCgizMQSEVGcPTB1C3731saok+GIUgoTFldhb0dv1ElJPAYHRCFjnwMiIiJ/bWvowB1vb8RPXlwddVISj8EBUcjY5yAYjLmIiMJV09SFO9/ZhHQMhgrNzLTc2s05hArF4IAoIszMEhFRkv3kpdV4etEOrNvVGnVSikrUZYgMDogiEvWPn4iIqBCF1IRnN7Hd3thRaHKKTlRliAwOiELGPgfBUmDURUSUNNeOXx51EkjH4ICIioJEVsZCRESFSqXTUScBSiks2NZY8n0DGRwQERERUcl7e+1uXDN+OV5aXhN1UiLF4ICIiIgoIQbSKhYl26mBNNbWFldH5NrmbgBATXNXxCmJFoMDIiIiogTo6R/Ah295D3+asTX0cy/f0YSLH5qL7r4BAMDk9XWhp8GJTXVtriZCW1vTgueX7nR9ntRAumg7UTM4ICIiIkqAjt4UAODFZdWhn/vuyZtQ2diJrfXtAIbmFTCzu6Ub5945PbLM83efWuZ42yv/ugi3vbnB9TnufW8zLnloHmqLsJaBwQFRRKKvFC4uCyv2Rp0EIqKisGNvJ3a3dJuuS6UVevoH8u7/7rrdaO7qx8Tl1dixtxPTN+4JIpnDGAcCrG4KLsPe0tUHAFhR1QQAaOrsC+xcUWFwQERFJQZNcYmIEu2iB+fis+Nmm677zlNLccbtU10d64bnVnpOy8bdrZaBSlCemLsdDW09puvOuXPGsNfXPb0co8dODiNZoWFwQBQRDrxJRERmRo+djNNvmxLoOfZ19OLnL69BV1/K1X49/Vpzop48zYrs1DR14a53Nzna9opHF1oGKjVNXXhm0Q7P6cj2x2lbBv8+795ZtjUkANDc1e/b+eOCwQFRRJJcwN2bsr9hEhGRd72pdKAl0g/P3IpJq3fh9ZW1nva/3UM7/YxV1c2e9zW6+sml+P07m9De4zyDPntzPXr6B7Crpdv2+v5t3vZCk5hIDA6IQpb0GoM3VtXi9NumorJIR2kgIioF87Y2Bn4ON808vUxk2drtrtR+w65WfP+ZMvzhnY0o0/sM5JOv03UxY3BAFLIk1xgAwNQNWseyrfXxDA6Sfn2JiLINOOgE7MbG3a2oaQq+Hf/rq4ZqJV5cVo2NuwubF0EpZdoX4CuPLsDLK3InLqvROyZngpQ2PZio2lt8Iwz5icEBUUSSXoNARETh+PWra111ArbTGlI7eWN7/FsmrccVjy4s6HjPLtmJ8+6dhc172pBOq8GhXWuauvHskty5CrY1aIVY5XVtBZ03bJlJ7iSijILj4EBERojIahF5V399iogsE5FtIvKyiBygLz9Qf12hrx9tOMbN+vItIvJlv98MUZIktYQ7qekmIoqLb/99iav+BG+s3hVgapJj8XZtyOqqvZ0YcNFmyUkm+8q/LvKarMBEVYjopubgJgDlhtf3A3hYKXUagGYA1+vLrwfQrJQ6FcDD+nYQkTMBXA3gLACXAXhcREYUlnyi5GGNARFRaVu+w769O4VrbU1L1EmIDUfBgYicBOAKAE/prwXAxQBe0zeZAOAq/e8r9dfQ11+ib38lgIlKqV6l1A4AFQDO8+NNEFF4GNyQH0SkSkTWi8gaESnTlx0lIjP0GukZInJk1OkkIio1TmsOHgHwGwCZbttHA2hRSmUGx60FcKL+94kAagBAX9+qbz+43GQfopKR9GY5SU8/xcpFSqlzlFJj9NdjAczSa6Rn6a+JiFzhZJiFsQ0OROSrABqUUsbp7cwKD5XNunz7GM93g4iUiUhZY2Pww2wRkTdRdZSiomaseTbWSBNREWHmPd6c1BxcAODfRaQKwERozYkeAXCEiIzUtzkJwG7971oAJwOAvv5wAE3G5Sb7DFJKPamUGqOUGjNq1CjXb4go7oolTx3Xm7uKa8IomwIwXURWisgN+rLjlFJ1AKD/f6zZjixEIvJPqd4x+aiwZhscKKVuVkqdpJQaDa1D8Wyl1HcAzAHwTX2z6wC8pf/9tv4a+vrZSntavw3gan00o1MAnAZguW/vhIiIkuQCpdQnAFwO4EYRudDpjixEIko2VbIhSTIUMs/BbwH8QkQqoPUpGK8vHw/gaH35L6C3GVVKbQTwCoBNAKYCuFEp5d+MHkQJk/RSCzYrokIopXbr/zcAmARtgIp6ETkeAPT/G6JLIRF55eX5FuQzRTwePN9uxfwMHGm/yRCl1FwAc/W/K2Ey2pBSqgfAtyz2vwfAPW4TSURExUNEDgGwn1KqXf/7UgB3YqjmeRyG10gTEVFIXAUHROSfYi51ILJxHIBJemneSAAvKqWmisgKAK+IyPUAqmFR0ERERMFhcEBEFIFpG/fgxCMOxsdOPDzqpIROr3k+22T5PgCXhJ8iIiLKKKTPAREVIOl9DqJQ2diBJdv35d0mKZf1f59bia/+ZWHUySAiopiJ+jnGmgMiciXKoObih+YBAKrGXRFdIoiIqCAsHHPGa0fqQrHmgCgi7HNARETknh/BBYdTtcbggCgiSS058RLUzN5cjzve2uB/YoiIKBRxKM+SWKSi+DE4IApZ0msMvAQ133+mDBOW7PQ/MWYSGnQREUUhqQVV+WTXCsTxubtlTzt+MGEF+lLpqJOSg8EBUYl5pawG5XVtUSeDiIioZI19Yx1mljdg/a7WqJOSgx2SiUIWdSnNb15bByBZnXq/+Kd5aOnqjzoZRETkA7PnYJCF+2yO5A6DAyLyJMxbbUVDR4hnIyIiKt2ggs2KiEIWx7aPRERERACDAyIiIiIKUdita70WypVqYR6DAyIiIiLyTRIGQIq6/1+cMTggoqLCiW2IiMgoiAqAYq5UYHBARC4x801ERNEqJHPOQqT8GBwQERERUfFLSHF/1E2eGBwQRSS5JRcJubsSEZEtFXVOlHJk8gdRdYhmcEAUsuSPmxzvBwmfc0REySM+5oSznwNJHXUoqmQzOCAKWXJrDIbz80ZORESlo9DaiuJ4isYXgwOiiCS9BoFV0UREFKawy6SS/ZT2jsEBUUSKpQaBiIjIjTg8/Vi+ZY3BAVHIkl5jkMFmRUREFIV1ta2uti+W525YGBwQhYw1BkRE5BSbcOb627ztUSehqDE4ICLyWWdvCutqW6JOBhERRSidVnh4xlbs6+iNOimuMDggChmrN4vfT15ajX9/bBHae/qjTgoRUQ7WRYRj2Y4m/HnWNvz29XVRJ8UVBgdElHhxq3ZfU6PVGvSl0hGnhIjIH/G6yybDQFq7at39AxGnxB0GB0TkSszy4TnilL4YJYWIyFQp3Kc4foY7DA6IiLKsqWnBnC0NUSeDiCg23OSv41RIQ+6NjDoBRJQscSyB8ftBdNVfFwEAqsZdUdBxYnipiIiiVwzBQ4APw6iDK9YcEJErUd+0koSXiogoV9BDejs5eiLuzxGVxjE4IKJEqmzsMB0NKPuG/8tX1uKJuRwTm4iSiQUyEYpjVXkIGBwQRSSpN/y43Csvfmge/vPvSwHkLwF6fVUt7p+6OZxEERFR7EhcHlwGcc4DMDggIlfidEPbVNcWdRJMxe8xREQUnLCGk45hHr9gcXxPDA6IIhLHG4IbCU9+oGIUPxERUcRW7GhGaiA5894wOCCKSJxK4L3wknyr0qWapi5M37jHe1oMx43bhGhERFRcTr3lPazVJ7t0om8gjYdnbjVdF8dHFoMDopAlvcagEFY3wUsfno8bnlsZbmIociIyQkRWi8i7+utTRGSZiGwTkZdF5ICo00hU7OKYOfWb18eu1X6ptPuLtq2+Y/ixY5wXYHBAFLIk3Yg7e1MYc/dMLK7Ym7POz/ta0qaWN/rJS6tx17ubok5GUt0EoNzw+n4ADyulTgPQDOD6SFJFRIHy4znoppNxnDPiccTggCgiSbhZbd7Tjr0dvfjj9C2+HC+ouMjrcZVS+J9ny7BwW27w49Q7a3dj/MIdw5Yl4KONnIicBOAKAE/prwXAxQBe0zeZAOCqaFJHRGEL+5nIJqjWGBwQRSRJ96VC0hrnG3BvKo0Zm+px/YQVUSelFD0C4DcAMr30jgbQopRK6a9rAZxotqOI3CAiZSJS1tjYGHxKiciVICY529fR6/sxyRyDA6KQJaHGwE/GpplxDhQoPCLyVQANSiljRxOzX4bpF0Yp9aRSaoxSasyoUaMCSSNRXPCuqfnk3TOjTkJeLy6rxvbGDvsNE2Bk1AkgovgzBjRuH1TpEAKCQk/Bh2/oLgDw7yLyFQAHATgMWk3CESIyUq89OAnA7gjTSEQBCfyem3UCr88IN4V5t0xaj4P23w+b77ocANDTP4Dvjl/m7cQRY80BEdkyu7E6vWkag4MwMuHM6MefUupmpdRJSqnRAK4GMFsp9R0AcwB8U9/sOgBvRZREIipA3CqJ78waNCKo9PX0D81lsLW+3fNxor58DA6IyBOnN9e0Yd6XuD0wKHZ+C+AXIlIBrQ/C+IjTQ0RFRBIyXETmWRlVahkcECXE1vp29CRwyM+gmxXt3NeJix6cG+g5vGIwZE8pNVcp9VX970ql1HlKqVOVUt9SSrEHIpWchvaeqJMQuW//bQnaevqjTkbkouqjyOCAKAGaOvtw6cPzccsb60M9b74bk7dmRf7nlv+5qAq7Wrp9P64Xezt6ExnAEVF8/Pc/ktlO3U/Lq5owY2N91MkoWQwOiCLiplS5s1cb3XF5VVNAqQmOm4kkvYxm5EvJik8xy5i7Z+IaQwe0UhuZiorbr19di9NvmxJ1MkzVNHWhN1UcgfnOfZ2hni+KCs441KrGIAm2bp20AamBtP2GPrMNDkTkIBFZLiJrRWSjiPxBX246zb2IHKi/rtDXjzYc62Z9+RYR+XJQb4qoWMXhhuqWMcNvl34v7y9ubUhXVDUP/p3Ez4vIyqsra9GbCj+jYqenfwCfe2AOfv3quqiTQgGL2/0+H7/u/6trWvw5kAtOag56AVyslDobwDkALhOR82E9zf31AJqVUqcCeFjfDiJyJrRRKc4CcBmAx0VkhJ9vhihJklSqXMg9zk3NgZf+CdnXkRlyotKSCVjmbGmIOCVUiCQ9E8MUxWWxDQ6UJjOrw/76PwXrae6v1F9DX3+JiIi+fKJSqlcptQNABYDzfHkXRCUiKTfPnv4BzN6stRd1k+H3kq8v5JIwkCCiuOPkkaWnpqkr0vM76nMgIiNEZA2ABgAzAGyH9TT3JwKoAQB9fSu0IekGl5vsYzzXDSJSJiJljY2N7t8RUUJ4ud/H4Rnh5EH1+7c34vvPlGHDrtbhHZKDaFYUw4ApjmkiIioIb2yhWVixN9LzOwoOlFIDSqlzoM1YeR6Aj5ptpv9v9u1ReZZnn+tJpdQYpdSYUaNGOUkeEQXE66OgSu9Q19bT7yrDH8RoRlGcLw5BHBFRkmTfN5N6G+3pH4ikE7GfXI1WpJRqATAXwPnQp7nXVxmnua8FcDIA6OsPB9BkXG6yD1HJSWohjDhIuPEm72YoU281B4VfyEy6uvpS6O4rjhFPiIjiKuyCIK+8dIA+4/apuPiheQGkJjxORisaJSJH6H8fDOCLAMphPc392/pr6OtnK60dwtsArtZHMzoFwGkAlvv1RoiSJqmly27bv7obytRlYuBvn4MzfzcNn7x7RgFH1FjFK7M312PGJo7dTRSIhN5TSVPb7N98NVEHH9VNXYn+Oo603wTHA5igjyy0H4BXlFLvisgmABNF5G4AqzE0zf14AM+JSAW0GoOrAUAptVFEXgGwCUAKwI1KKRbRESWBSa7d8SRoaRd9DhzeTocFKD7XwHQFWHPw/WfKAABV464I7BxEpSaptbBWtNLqJGct7UVRODZ1Qx0OP/iA0M6X5I7ktsGBUmodgHNNllfCZLQhpVQPgG9ZHOseAPe4TyYRxY2T+15lYyf2tPb4esxsudW+wY6O5LcNu1rxsRMPjzoZRIkXh98zxdcPn1+Fl/7n/KEFCfnCRBH8coZkCsSczQ34yK1T0NGbst+YitZtb27AL15ZO/ja7l7s9F49rOIgxqWGTmpCNu5uDSElRMUrxreAREhyCXeSxfmyMzigQDw8cyv6BtLY3tBhvzFFpqsvha/8eQHWepiBMV+m3OqeZ/cQcvqQMnZybunqc7SP0b6OXtS1dmNHY6frfQGgu28AFQ3tnvYlInIjxnnIQJVK0BLHAi4nfQ6IKCF2t3TjhCMOtlyffbNdV9uKTXVtuGdyOV754WdcnSuI+7bTzsvGzV5aXmO5nZVP3j1z+PFcvpefvLQaM8vrsfmuy1yfO5uX0TCIiIqd3W15ZjkHdwgKaw4oEO8qLPEAACAASURBVCUS8MfKm6t34bPjZmNZ5T7H+9hlS/0YJtTI9mvhNDjIs10Y373F27UJalJ20YyTtDA2IPJFqZQ0k+Z/n1tpua5/QGFWxMFDkr+NDA4oUHGsLosLv4daW1XdDADYvMd5cxfbPgB5HrZBfLaORyuKyW23sb036iQQlTy/CzFKQVLjqMnr6xxve/2EsrzrnTxHSvWrxeCAKObSaYWJK6ptt8vcw9J57vqWq0K6AdoOZepDzUEYMsOd/vjFVXm3Y8UBERElDYMDCkRcSnbjzGlb87fX7sZf52y3P14ARRxhl8gl7VvT1tMfdRKIiBInaff6INkXdoVfhMTggCjm3GZAPZWq+32nthyuyGY3h4nP2+fA0RHig00iiPyRtN++pVK4JURd/eswCUF+FHG+9TM4IIqI37UrmRuNm6OGfW+ye8/ORysK/8GilEJl4/ChedNpu33Ml3cHOAszUamJcR6LChF9/FCyGBwQhSyokuJMM6V8pe9u77WZCbqM+wV5v3aa6c8XROTrc1GI8Qt34OKH5mFd7dCcEF5HR8mMdgQwY0NE/mps78U3n1jseMCEjbvbAk5RaYpB5YhnDA6IQhbUcHuDNQc+Hv7WSRsAAANOi/Rt2KbNcYdkD52uCzztan2iuJ37ugaX2Y9kytEwiMKS5MxYPm7f1/NLd6JsZzOeX7rT0faTVu/ykCqy5/yDm7yuDg3tPQGmxR0GBxQoTvAUnrCudJDncXorzbddUDUH++m5eOPxjX939aUMW/N7T+RWT/8AUgM2bfVMFFOA3T+QRl/K/TWg5OroTeHGF1fhuqdXRJ2UQQwOiCLidx52qM9BvlJ183Vu2vAXkmz7eRUcHieCEsJM/sN4bmMy7p+yOWcfq3QalxdTxoaoEGfcPhXXjF8edTIidec7myI5b29qANc+vXywKampIqqaidM7GdA7r+1u6Y44JUMYHBCFLLA+B5LpcxBOWoKoFXJc6h9Bn4P9JPf4aUO7oqYu56NKxenBRBQnS1zM8F6MVlQ1RXLejbvbMH9r42BT0jgIYuCJKAtj5m5pjO7kLjE4oEAUUQGD74LqczB4/CCOaXJQLzduu/fuvFmR9ZZ2IwhZKa/L3ylvqFnR0LIBHz5LNr0j8gfn1yncmpoWtHbHY/6WYstH9HloMhcVBgcUKDaZKJzTS+ikQ7LdvbahrQc3vrgqq/28+4DD60PaaeCUryOw3RH2dZiP4PHHaVvy7jdUM2Nec2CWdkczJPM3QlQQBtj+uu+98oKP4UdBUFfAQz4HHXskObhhcEAUsqCHMt3T2o1xUzYPy7g69cC0LZi8rg7vrqtzfL6M6n1deGj6loKGUnXe5yBPzYHNQXpMOvsppWz3288k+PI6iFPQtUdEpYg/K3/0WnWIDrkk4853o+l/EbY4fm9HRp0AKk5x/LIXu8x9e8ISbfi6Sz56LD41+ihH+2Y+L+sOtCpn22w/eHYFttZ34D8+cZLtebKJuPvO5N3U5jgHjDAvE7E7v9loRcYhXs12t+4ATkR+Ye1b4cJ+ZicljxDmdytO32MGB0QJc+EDczCQVhh7+Rn42tknDC7Pvq+Y9hPIWma8Ga3c2YzXV9V6Tlf/gNLP4f6uL9AyzE47E+fbzO4YI/YzvwPbnVkGOySbn2uyg9oWIiK34tCXYvTYyXjfASN8O9Ypxxziy7EoGGxWRBQRryUn1U1d2NXSjZ+8tBr1bUOTphRa6jCzvN7RdlbnGRzqM8++dg8554MV5WtWZLOvyUnSCmjvyd8JT2zmOfAqqGZmRER+8rMPwI69nb4dy4vlO6IZFSopGBwQJVi/YfSDsDrl2eWHlXIf+Ax29jVZ99B0k47CBdQcWK1dXd2Sd7+hPgfmzYpMz+VgngMionymbtjjaT/eZpKlq3fAU1/BIDA4ICoSTgqgvY8iZHZCm9fmCbA5T+4Gf5ld4eowQXX2zfQ5uP2tjYPLvN/Hh3ZkvQGVmt7UAD5y2xS8tWZX1ElJhJsmrnG1PSsj48Hs8VDR0I6Jy6uHb6dv2DeQxripuZNpPrO4yv/E2WBwQBQRv+cJKOR54CmPa7mTtz4HbvbM3+dA+7+lq2/Y9erpH0BP/4CnUvvO3hRSXidQ0NNipRQf5CJykIgsF5G1IrJRRP6gLz9FRJaJyDYReVlEDog6reS/vR196EulTWcVL0Q8ylyTilcvDJc+PB9j31g/bNncrQ2Df09anRswv7N2d+DpysbggALB24w113lBp7nHrO2c7CYWf2dzEsgUUnEgJk128snXdEgphZqmLpxz5wyMX7hjcPnHfz8dH/3dVEfHz3bWHdPw0vIax9sbr31ZVRPOuXMGpm5gh2WDXgAXK6XOBnAOgMtE5HwA9wN4WCl1GoBmANdHmEZKCHFbumDQ2t2P0WMnY8YmZ32uSkEJlldYenD6Vs/7mj2mzGqbn19anbswYgwOiEIWVODkKHPu58ktTph3EjaroUwhedfnHCfPurQCapq6AACzyodKZPoG0lp/CBefgNdmD8b3sba2FQCwtLLJdH0pTuCkNB36y/31fwrAxQBe05dPAHBVBMmjElLRoH0NH5+b23yRilvUfb/Mzt+XSqM74MnfnGBwQOSTpZX7UNvcFeo5jSPdeGmeYrw3ra3N3yE3787I36nY4yEtLavcZ7kurdRg4FLoEIBPL6oqaP9hQYDh82HNGiAiI0RkDYAGADMAbAfQopTKTM9dC+BEi31vEJEyESlrbGwMJ8FEMTDm7plRJ4F8sqYm95nb2t3vuYbbTwwOKFCl1J766ieX4sIH5jje3kupRb5hLwspgRYAiyqsM9xO0mo8u9Xmlpl1k9mHAQwbqtXoF6+stUyHFhvkqYlwcd3Xmty8nRgWCNhcvFL6jRgppQaUUucAOAnAeQA+araZxb5PKqXGKKXGjBo1KshkUgAKHTRAKYU5mxtiM7ILUbFhcEDkIyfPqoI6DufrkFzAge2SbbreQ7MiK0NNhofv3NGbyt3YRlqpoT4M7pPii6bOPv38xlGJhi7Y8GZFpU0p1QJgLoDzARwhIpnJOU8CEH5PvCKhlEJda3fUyTCV+f57neNj8vo6fO+ZFfhnBKO4FKuom9hQvDA4yCOdVmiwKLmk/AopGdrTmuxr/stX1iI1YD2yTZR9DvzYx/JYToZSVVqbyn0dvab7Zg8IlBpwf7XSyvC+zGaJdn1E92znPyjxhkUiMkpEjtD/PhjAFwGUA5gD4Jv6ZtcBeCuaFCbfC8uq8Zn7ZmO93udldXUz1rltOmjh7bW78fAM7x01C1Xfpt0/sptxFvK7SnLmuKKhHaPHTsaWPe1RJyX2rn5yaWjnCmpY7TAwOMjj8bkVOO/eWYOdG8k9t01d3ltfh/Pvm4VFFXsDSlHwXl9VO9gJNUxBNk/Jd5Pr6R/eecruAf2zl1fjkxbtZrP37c8TZFnRag4y/R/MZkMO74ZtPNXTi3aYLi/RZkXHA5gjIusArAAwQyn1LoDfAviFiFQAOBrA+AjTmGjL9BlgK/dqHW6//vhi/Ptji3w59k9fWo0/z9rmy7H8UFCTyiL4/U1Zr02S9saqWnz54fmYv7Wwfjh+3CETnC8mACPtNyld8/QfWF1rD04+6n0Rp6Y0rNzZDADYtLsNF5x6TMSpCZbf904nVfReb9hWu83cVI8fPFuGaz/zIUcPaAXgvfXWs31mp6/PQ3CQ73gA8JvX1hV0TD/SsL2xI3dhCVFKrQNwrsnySmj9D4hcS2qGtKKhAzXNXbjo9GMLOs6eth5sqR+qPVixoynP1sGoaepCd3/0o+1QYVhzQKEZSCtMXF6dt8lNRjGU5lhx+9bifCkyAfSzS3YOltLnH8pUmf5tFVj0pdwHB0u27xscntAsKQu2RV8r9chMY6lrnD9houD4dZ9P+vPii3+ah+/9c4Xvx12SZ1S3bMZ7pdlEXE597oE5uPTh+Z73LxVzNjfYbxQhBgcUmheX7cTYN9ZjwpKdltsUWvKjlMKMTfWxa+t3+Z8X4NZJ6+03LEBBHZLtrpcy/XPYObv0sZmVctbW0mwT47IHp23x1D50YcVe3KJf66i/B07OnvSMDcXHvo5ebNrdFnUyPDHWpt34wipH+9j9vO+bUo6bJq4uJFmhC+uWlT3je0tXv6/HT8JAUlH2/freM/4Hg35icJBHzPKXides33xauvoKOs6OvZ2Do8Fkm7iiBv/zbBleXuF8Nls/VO3tzLu+vK4NLywbPgui3xnX7NJ3szxn9s3QKmOavdhJStt77EcWMo4+ZBZkGPsDPDan8EmJkt65nYrPUwsqMXrsZE/9aexc/ucF+MqjC3w/rt/Mbn2XPDRv8O/J6+tQVmXdJMYunlZKoasvhb/Pq8Rba5wNeFUMj/v1Jn3drCZy/FNWh/IXl1kX2jmRff0enhldh3UqHIMDihUnkfxFD861nE+gTs8M1oWcKfzCg3NDPZ+ZsEqgxeLvTj3jn+8zvGb88sG/hzcr0pf5kD6j3REHB04CQFYclJZMR96uAGZBbWjvtd/IxkBaYYdNYYdf8t2zvvm3JfjDOxvR6OI9pdIK6bTChMVVOPN305ylwfHR46/S5HO7aeIapNMqpznvugAGzVjqohlTHNw6aUOgx09ywMnggELjpqDcrnOtl/Hvi52Xh5zTz2RYXwEP58kwPuiN1c6Dowvp55m8rq6As8RLqQ9bSsny6KxtuOjBuYP9doLg9Dfxz0VVuONt+wyc8d63dMc+TNlgPehBsdjX2Td4v7QrGLrm6WU49dYpuHXSetPaBQCoKHCQhMUVe0MdJjQJ2rr9baoVJgYHFLp89zG/WtqUYjtuL++5kMstMA/iHAccJmfPLLnxRWdtjouB14mgiIKwXB/hJow5fpyMcJZvzpAF2xoH05uRPVdKRmogPTjscmVjB0aPnVzwkJ/5vLe+DqPHTsauluAmontpudZ81u4esqhCK9F/YVk1rnl6mek2NU2FpXNZQCMjXTBudiDHDcMNz62MOgmeMTgAMHdLA0aPnYzyumR25Ioz4z3r7bXOR0AoNLuUhP4ivg9l6mQoUYuTepkhudD0m80SnITPzQ2F4ntP5L/W7n7c+MIqtPrcKbQYXfzQXNz57iYAwPbGTnz770ts9/nZxNU49dYpOOP2qQCAsiptyOx31g71R6jTM/HjpmzG6LGT8WYBI/YAwGsrawEAm13kK9zWmO6J0SStQY0CF2RwRdYYHACYsakeAFCmj7GfJHtae7B4e/RDM9oZSCtsbwynHWsxy5fP9FRz4LhZUe6yd9fWDSvVy5zfU2ZY3/cbTyzGUwsqPRwguVhvUPyUUrhn8iZs3mOeUfznoh2YvL4O4w0T5SWJUsq0f82U9XW+91+oNHmO2GWS38zqlDxdf+a/qmfgAa2/xrLKffjbvO0AgJ+9vAbPL3XfSXdvRy/eW++tWeTk9c46Txeqq4/Ncik/Bgd5JKG2/8uPzMd//8O8mjAI62tb8ZFbp9hWORdaWpqEa18MChkx6fVVtXjO5OGpHLYoNqs5AICHphfPKBdK2dew8LuefAu2NaI+zz2xqbMP/1iwY/i9uohqlG5/awNOufm9nOU/emEVLjIZrMHNbWfaxnr88pW1ebf5t/uHBqhwcveZWV4/+LexSc5WwwRiAPC7t9x3WP3+Myvwfy+sQqvD9uarq4cKJb308xi/cAdqm7scb9/S1Y+fv7zG9XmotDA4SDinNyC//HPxDvQNpDHfYRWi24xPocN7JimfZTu1gFI5JY3Z78/t+zWesqd/wHR0C/P9/M/JsKMuxV3/QBp1rUPNGiYur8Yn7pqRc5+6ZvxyfO0vCy2Pk7YIhDM27PJ/5Bgv+lLpwYmzVte04Kt/WTDYTj+f55dWD+7vROZyOH0+vL6qFh///TQ8MXe7/bFV/uPu7Sh8RKd8apu174uTyT4B4OuPLx78e2t9B7Y2tOfZerhHZ23DXe9uGux74NS0jfX2G1FJY3BgxMbBgRn2MHXwREhSJt8tpw/ECYurcNkjC4Z1uHPzDb110gak83Tm+9Wra3HzG/5PzOalqZKx9K7YStLtrkexvd9i0T+QxvefWYHP3Dd7sBnGzZPWo6mzb/Az/cf8ysFJtvINI/qpe2ZarltX24KZ5eHNlrq7pRv9A2ls3tOGZZX7cO975YPrMk1sAeCP07Zgw642bN7Tjvq2HrT32BdEOdnGyM1Xv60nhfunbnZ1fDNj7rb+LILgvoAsmHQQuTEy6gTEGX+khXN7DUvhkju9Jhv0mU53NjmvMjbaUt+OnU1dOOWYQ0zXL6oYqv2J4rteCp91qbzLYnTzG+sHO1n29KfxvgNyt7nHkLE205dK4w/vbMy7jXGivkdnbcPJRx6Mb4052X2CHfrsuNn4zzEn4+WyodLms044DFeec6LlPp++dxZOOPygwNJERPHCmgMKRHaTEbdZpEKHd2SWTJMz07HHKMDNbk5H6DHOhmz8uIutIN2u+ZSTUaYofK8ZOqsCWk1C5iurAEcjC03buCdnZnQ7v35tnavtvZi7dXhNxdLKfbj73U1IWY0DCmcTCvYZhgs109LVh8semY/KAsfUz4f3fqLCseYArNYPkuuag5LqyOzszTp9S24DKq+XutB+IUPHGfo7UR+bC44uVbG++SLy1IJKvP+gocfljS+swrRN9hNtJSWjmmmzftU5JxR0nM/cp41JXzXuCtP1M8sbsHlPOx6bU1HQefJxe38K6ueXlM+eyIxtzYGInCwic0SkXEQ2ishN+vKjRGSGiGzT/z9SXy4i8qiIVIjIOhH5hOFY1+nbbxOR64J7WxQ0tzdgtxnBZGXy3Qnzvfl1Llc1B85nQRs0vM9BcX34bJ6YfI/P3Y6Zhvb4UzfuKfhzjWOH/DxdlAIRu996AOlhzSAlkZNmRSkAv1RKfRTA+QBuFJEzAYwFMEspdRqAWfprALgcwGn6vxsAPAFowQSAOwB8GsB5AO7IBBRUvNw+AOP4wIwz22Eysx5Mxu29Zm7sHuhOD1sKnzUrDopHyoecs9Ws4qUm6ICZGXKiwtgGB0qpOqXUKv3vdgDlAE4EcCWACfpmEwBcpf99JYBnlWYpgCNE5HgAXwYwQynVpJRqBjADwGW+vhsqWGt3P5o7+2y3c9+EJR5VvY/O2oYXXbYBDortKDY+nCOIfEjQzYpK7bHOmgUqVXH/rfOnSaXKVYdkERkN4FwAywAcp5SqA7QAAsCx+mYnAjAOulurL7Nann2OG0SkTETKGhsb3SSPfHD2H6bj3Ltm2G7nNINotlm+PYPOKP1pxlbcMsn/4TuD4GwiMe8XzM2+7rZ1tl13/wBunbQerd39sW5GZmxO4tZtb25Ar8nY7187u7C23ZQMMf5aO5a09xCXDD2Dfkoyxx2SReT9AF4H8DOlVFuekmOzFSrP8uELlHoSwJMAMGbMGP68Eir7xuj6Rmn4fi2u2Iu2nhQu+9gHCk9YEXPyEPf6wHK7m5PtJy6vxgvLqnHgyBEYlvoY5UZOOPwgbNzdZr+hheU7mrDXZPz7d9buHvy7FJpXFQM/vpZmn3RQzYy8HDWsID3Qb7zLgwf+nmN0PyNyylHNgYjsDy0weEEp9Ya+uF5vLgT9/8zYaLUAjIM0nwRgd57llECFPNDy7Wl2X//vp5bhh8+vdHeSIii28SOzH02JvLNrn2nDHefMsR8pSxfBd5G8iWuNWGy+kjG9Pn6I832NyI6T0YoEwHgA5UqpPxlWvQ0gM+LQdQDeMiy/Vh+16HwArXqzo2kALhWRI/WOyJfqy2Ivrjd4Kx29KUzfaD/MXiH8anduJWGX3JMwHh1eg7hCgr9vPLEEa2tabLez6t8Zp8/ej6+53SFik1GjUPFzD47X29ftb24o/NwFH4Eoek5qDi4AcA2Ai0Vkjf7vKwDGAfiSiGwD8CX9NQC8B6ASQAWAfwD4PwBQSjUBuAvACv3fnfqy2CiWe/VvX1uHG55biYoG/yea8ToKhOPRLYvlQ4iJnE/LcH3belJmi00F8blkAkyBxDYAV1BYUrnXfsNCzsHvfNEyu1/G9bseqgC/9HEpsc+8RX7clES2fQ6UUgth/f2+xGR7BeBGi2M9DeBpNwkMQ7ENe1a1rxMA0N1nPVOlV66HJvU8XKbH/YrsswQKC1p9m+fAQSq0a++i4/KwfeOpvq0X9W25fQb8FI+sDNny4ccU1+95htn9OpARzzLH9v/QsTEYHDAapARyNVpRsWrr6Y86CUUn+xljzFzmv1cyq5Qtbx+NYROJBZ6UvNwGjmm9XZFI9GmPUtBN9IicCvubGETG2e3PqRgLlIgKxeAAwFtr2C/aKbc3Uq9VvKVww/aSJyz0wefm83hi7nZX5+0fcDvhnbliK2lj3r90FcNXuRTuxUQ0HIMDcsVrZp99DoY4zTBktss7L0RAZX3d/QO4f+pmw3n8N3wStOLNgNh9RiXwlS8KnoYGDei4FI5Cnkf8XVOSMTgoQn6UVqXTCq+U1SA1kDuBkxth9zlIgqACoIIuWVai/Bi1w05miM/mzj7saesZXF7Mn72ZUgiIi4Hf38ugP3Yvxy+Gn55S8bqHxCgpRI4xOChCfmQ2XimrwW9eW4fxC3cMW+55tCKn25VQRsmPtuZuD+F5ErQARyuq3Nvp/8FjxP7aldCXnkoys5gaSOOHz61EeZ02oWCQI/nw10RUOAYHZKq5S+uk3dTVN2x5dhOJdFphTZ4x7U1nBHXwSCj0oRHnB0SYpVpxGdbPTGaeg/2yrkexZZ7sgoNSCoiTrJi+l2F/5bbUt2Pqxj2DhU3rd7UCiEcJf1hp6OpL4X+fK8Oe1h77jYkixuCgxDy3pApPzt9uu51TTy/agav+ugiLK4aPBZ9dKu60lDzOmdk4Cm9+Av8/l1nl9QCGAlGiYhOHzC/l9+T8ykCOm/3Mm7yuDtM21uOP07YEcj4iP9nOc1BKiq0Ez+zBdPtbGwEAN1z44bz7GjujDjtmVvnZ5j3tAIDalm7T7b02nSnmh2qU37M4fcV36yVoO7KaFRXbaEWpdP5+O3H6TMiat+9lcX2Xk8DtM+fttcGOVlhktzMqEaw5KCGeboI+NYkIq89Bkm7Edm/VSfMr44PQLPPi9HI2tuef6KvYAucw2U2ixmtbmpIyv0WS7qkZUY5+lsTrRZSNwUEJ+elLqwM7ttX9MPP4cztZV6E39zg/d+P48Dj3g0dGnYRBMbw8gWJTutIS55qxYvgmFsN7IIoagwPKa1V1M2qbu2y38+uGzBt7fsYhPwEnfQ6c9f045MAR+Y9jmzLyKs6BbFBE5GQRmSMi5SKyUURu0pcfJSIzRGSb/n9solZPjYqcFITEN1bwjdV3vKjnN7F4z6+vqsXrK2vDTQyRSwwOikRQVdQrqprxb/fPyVn+69fW5U+P7QIbCXpmeL32tqPYuLxohWQydrfEZwSNUsgsGZVgbAAAKQC/VEp9FMD5AG4UkTMBjAUwSyl1GoBZ+msKSdg/vVL4rWcCIGON0T8X77DanCgWGBwUoShKIi2bFeVJS/W+LvT0DzjevpB0BCnSDsY+9QlZmDXaVO5xSjQLS4FQStUppVbpf7cDKAdwIoArAUzQN5sA4KpoUhiMuGeG/fqVZ9/X7dS3+Vc4EdatqmpvJ1q7nY+0FvOPnmgYBgdFIu55t+xScKUULvzjHPzw+ZWm23u9kUZxGYI6Z1yr3IP/rsXzfQel1AMvERkN4FwAywAcp5SqA7QAAsCxFvvcICJlIlLW2NgYVlJdi/Kb/E6eASiC/so96HK4zi8+NM/X84cRhH3hwbm48rGFwZ+IKAIlGRxsq2/3daz/QvSl8g9zGHf2TV/M12ceTnO3DH+wJ7FzZrSZO8NoRa629noWIv+IyPsBvA7gZ0qpNqf7KaWeVEqNUUqNGTVqVHAJNPCS4czX+biYf1NuStQBoL035ePZw7uyVfus++NlnmVxry0iMlOSwcHXH1+Me9/bjNRAtOOPV+/rwkdum4LXfOicFPWDJuchmJWg7Pyz7TCeHu+okTQr8rxf1J+ac0ppVf8vLNsZ+MOOD9PSICL7QwsMXlBKvaEvrheR4/X1xwNoiCp92fyoyVMqefVi/D16k3nmZS4fryMlSUkGB119WimF30PKjR47Gb97a4Pj7bfWaxOI/erVtWjq7PM1LXFhVahuWdqenPzyILcVB342F/Krz4ETP5hQhlsnbUBda3w6LxeDUmxVJNrNdzyAcqXUnwyr3gZwnf73dQDeCjttlGx+/57W17b6e0CU5m+ekqUkgwMrfjQPeXbJTsfbGmOTHXs7Cjpv3NstZ6cu+3VdazdGj52MVdXNAJJVumZWA1Db3IXNe8xbSURZY+D1a6KUGgxgB9LBpv+AEaV1W0pSDZKPLgBwDYCLRWSN/u8rAMYB+JKIbAPwJf11PAR0UyrlEuU4z/mQ8bXHFuZtJnX3u5vwl1nbbI+TgLdKNGhk1AmIkpahdv+L3bmvE98dvwyv/fCzOO6wgzyf33izsGnhNMyExVVYvH0v/n7NmMFlcc1emKVLkJtJXVSxD0D+NpzuzhveFTHLcN80cQ0AoGrcFQUff21Ni/W5bfb16zqE+f06cGSJBQdx/fEGSCm1ENY330vCTItTnuY5yH4tpfF5F1tGuDc1AGB/03VPLdSGJf3JJacNW14CHzMVsdJ6Cmfx+uN9dslO1DR14+011qNBuJV28cS44+2NmLax3rdzhyG7ZiP7/Zo9RKPSP5BGXWu34+09P+xt9nN9DYrkiVy5tzPqJISqFDKLpapIfpKBCeLyRP9z0t6Vyup0ENfR54jMlHRwEDXjzcJNcGAmsgyGwzbvUSWvszfluhnMLW+sx2fum41OhyNouJ6sLMI+B37UJMS9CRtREvBnBOzt6PX9mErF49pubyytQg4qLiUdHHi9gQRS2hHyzWx3SzeeWlDpap+api68vsp8ZCUBNf9ZewAAIABJREFUsKq6Ge+u02pTbJu7+F1qbrJfOq1w1h3TcOuk9a6OMbNcq5VxOpGP288ukW3ME5jkpOClLS3xqE0I91tndY9saPc/OPjVq2ttJ3UMilIKbQ6GcY1D8EKUT2kHBx5vkL79ro0Z2UJrDlym6voJZbh7cjlqm5238b/26eV51//H44vx4xdXa+nJej9uS7B//vJaXPjAnJzlDe35R8oxnifz5ytlNXn3sZLdWa56XxembtiTe05PR/fne2S8jmZ5Dr++q3yWBYc1McngbZ4D/9MRNi81nW72uWb8MtfHz6fb5ezMfnqlrAZ9Fh0Ii+G7QKWjpDskF6q8zvGcPaaM9wovA8BUFdA2u71HK91wky/J7GNktXtmuVnGJ7tTXjqtTG+c1U3DA5fTb5uCU499v2X6Tr3lPexnOJDXTFdmr+wkXfrIPPT0p3M6Gbs9T74H50PTt+Dv8yux9e7LXR3TSk7afMiHMisLfOS492NrfWEjjGXwepJXSqlYjvjjJkkLtkVTyh+E7Ek9AfY1oGQq7ZoDh09lq5/2G6t3uTrfqupmfPz309DSlTungZeagy88OHfw76ALH5fvaMobwOTMgZanpkCp4RmiCUuqHD1MelNp7DGMsf/k/O34xhOLB1+n0sq01MbtpRnsR5aVpp7+oWPXtXZjW307lu9o8jVz95fZFa5mzQ6r0LmjN4VdLd2hnjPOfH3g83qWNK/fpa317Tjl5vcwY5OXwSnMz2lVmEPO5Ls3vrPWvwFMiIJWkjUHUT2LH5tdgbaeFMqqmvHFM48bVuKT9nHs+CDe37f/viT/OS2CAbO0DCg17CHU0N6LI95nPkxca5d1+81739ucP0151+bZz0Hu9zP3zR78+7nrz/N4pvz8eCjXtfZg1KEH4tCDzK+vF3HuM/GTi0/FX2ZXBH6eA/cv6XKVkhR08xq31lRrQxxP37gHXzrzOMvtbpq4Bht2+T+RVynx43OcWR6byb6JbPEJ50EQJadBzCuljc0cjZxrlNWMyOnb/ey4WQWnwey23uxgRmqnD4R9Hc5nt87Mzg348z2yO8Z3nlqGKx5dOLR94aeMtWPef2Ao57kpa0zzQsQ52KIhnoL1PPuE+bn/Y8EOR9vFsYlSMdpUYJNkoqCVZHCQuf05bY7t1y08Zyx/w9+Fdkg2840nrEv7/Z7l1uqZYva20i6GmuvsGx7gOE211nTJeutz75rhW/Dk5nn64rLqwb/9+sz7Uml09w1Yvtvqpi7saunGo7O24c8OZvK0E+dmRWHlbd5/oH+VrnG+nuSOMfi35u5LWrW3E6+tNB8lDohnwJ/TzDSWqfSf2ftkvEV+MOvzGSQ2K4rwvMabRqEZBLf717XmH/WnUPkCr4HsTge+nC/3gHbXpH9AwSyPN9iZ2mEi93Nx9zcGBNc+vbzgGZQVFK59ehm2N3bip3lKs3/43Eqs96lpQZwzs16ew0cfcgD2OahJMjr75CM8nMlcjC8nuZQKoAr4a39ZiPbeFL75yZOGrygo02meTo6cZW17Ywc+cNhBOOTAkYHMz0AUJyVZc5ARVWmGWV4yiTdlJ2k2u8b77ycmMyR7f9Kl08pTsyy7MzrusO4i6UE0H1ta2YRGm/HCoxzeL+7evPEC7Ofy63fQ/iN8OffB+4/ARacf68uxKFhRlQC3O5yMMSgs+QaaOvtwyUPz8L1/rkBrdz/G3D3TdLsEPsaJTJVkcGDVrCifL/1pHh6ctsWX8w+1hTfOkDy0fuXOJtMRjfIeM4JAR5uJ0r4E6hcvr8H1E1YMvj7m0AN97Szcn057qjmwPom7zd0ENgNZw7aOHjsZ9W3ea3GGvcc8bziJwacneXIyVsPgRpn5Kb/rMnzg8IOiSwAFKoyvlpeftl+3g8wIZtlW7mxGymK8/yT68iPzAQDLq5ocTXJmxNiKkqgkgwOr+2Jbd8py7oBtDR14bI6/o6AMm81Xv1srpfCNJ5bguy4nhoki72c8pWUGS2lDvm7YNbwDlp+Z1dRAbs2BsvjbCZX1vx1XNQdplfNZLdm+z/kByLPPnXaM6fL9RNgRk2x5Gq3IyffKsr+WynufHCzkirBhWllVk+nyrfUdeHjm1sHXr+fpM1FMzD4J3lsoiUoyOMjI/iE/PHPrsLkDgmJ2rxgKDrTX2ZlpJ+6futl0Bt+gGB9cbubaMutyUMj9s38gbdq51+6hmb22qy+FG54tQ4deje80gHHTJMWsWVHmvRtrppyeu8vQYTvofiQZca6FyPdRWPUN2U8k1u+JksP+VmDsaJZ/y38sqMQpN79nuX7znnYAwBur3M23E5Yte4YmCZywZGeEKYlWb2oAD033p9UBla6wg8zSDg4cZgiyPxK/SmrMRivKPvKe1h7c/e4m29GFOnpTeGLudvzw+ZWhZXTsAgCrbbQSMf/O+eySnTnHE2MalPkoItnXadrGPZhumFDIec2B8x9tZ57RTIw1U077Jhjnn3h87nbL7fy8scQ5G53vbVoFcSzYIyf8+p5kjvPe+rq82+Wbx+Xap5dj/ELr4UlvmrjaU9rc6h+I890gXB29KdPn2oTFO0OZe4WK2/ytubNvB6mkg4M4ys6w/vLVNXhq4Q7L6tsMN7PqBsHN0HXKZn0+ZpPF/WnG1pyaA4XhmdgzfzcN/VltYLOPlM66hE4DGDeBzpPzK00zqVOyMgpxLskOYtjdQv3XeScP/v3nq88x3caq5sBrpu/+b/yrtx2pJKypacHNr6/LWZ4ZPOD372zydNxllftsMwpvrfE2G++yHfmfM9l+9epay3XFGnRb3f4+dsc00+Wp7AcLkQeVjR32G/mopIMDP7I4L6+o9p6RMxu1KOt1pmTG7gxRZNhOu3WK5bp8ydmyp93zxbfazbRZUdaynOAga5fcAMPZcH+ur73JU/NHL6zKOqa7Q4apvs39MH5hZRQEgsMsZoPez6LqIN9QtKccc4jlOqvzEAHAVX9dhN1ZTf38GALznvfKCz5GV5/56GV2o56ZuUzvrFsqBlze74s0RqKQsVlRDMzd4nya89++vh6r9Gns3TLr4OY1jx9VZtLqtIOdek02mLiixntgZrGj2fu3PUd2PwmPNQWuJ5TLHsbV5EevBRz+3QyK/QFlvKTWQYD5vlbBwaEHjsScX33B8pzsaEjZ3H4n+gfS6Dep9e1xOfTw0kp3gxq4GdrYriN2pu9DthmGJprF5KI8/RJnlue+5z0eClOIsrkdJatQJTkJWoZVZvD//XMFlt9yiem6lq6+nP2uHb9s2ARUUzfU4a53yzH311/A/iOG4i+v8wJk3PnOJsuSJ2PpdRSZluwHiN1bNa5fvqMJZ3zgUEfnsQxGsk64u6Ub1fu6bI41fJ+cEiGLk+U0R3IZ0cW4UiAwgmDf9+DwwGIdBIyw7JBscVCbn5HbuREoGdp6+tHbn5thX7Btr+/nOvN3U3HpmR/IWX7G7VNzls3f2oiW7n6sq82dzPDqJ5f6nrYMxsCFKa9zP7gIUbbtjeYjaQalpIODfLmVXos2/F9/fDE+/5FRw5Z19g3gvilDncdue3MD9nb0oaWrH6MOPdDyHMabbiZznS+f+fQi6w5ow0cOik/207JpjmH5yp3NOPnIgx0dr8NiQqDswvu31uy2bXdr16zogWlb8OC3zjbZr7BmRdnbm2Uy49iuvxAiEsp4u4I8QYBFbl4yQ5nmTMw33AEj9kOfoWlaoUH4p0YfWdD+5K+Wrj6cc+eMUM/ZP6Cwx+E8J9c+vTzg1BBRfIWbJ2CzIpd2WMyDYGd9bSvmbAmut3lrt3mm2XWTF9/YDCOatbqpq7AqMyeZabuKgez1rzkcmzvfXD83v7Eeda3DJwrK7p9mVm0f5z4HcWQMNq0y7VbNI7S4IPeC22X+C605uPWKMws7ABWkracfv3ltLTr1Aofqpvw1jU54mfhr5c7mgs9LRMUt7DxBSQcHTkfM8eMz+ebfFucsM+YtMmmxyuTa5X2/8UTu8QFg3JT8ndeCKsw1DiNqJvt9HnpgYZVYTt5H9jlzOiA7vBi5oxxZ7/fS8mrc8sb6vOc1M+DzEIFhNw24+Ixjh58/pPOKACN86nNgNwJXodeUrTWi9be52/FKWS2eWVyFzt4UHp1V+HCT59w5Aw9O24IaHwINIqKolHRwMJBWeGpBZSjnMmZAMn+alUyajddfiKkb80+KFtTsmvmOeuhBI3Pe5yEHjijsfI76c2TvM/y108g8e7+UzY525zXLZJ5953RniYmpsDO+wzokWwUBeYIGs99izvwmOZ+btsVnP3y002RSjBg/zj9O22LamdSLx+ZU4HMPzPHlWEREUSjpPgdvrtmNuydbl6z72XZ//xGCTGdzpYCP/34aDjEpLTeesamzr+Cse6YpxXNLzWeoLPQtWu2fuXZmqz99ylE5y0aOKCxOdZKxz21WNHzBgm3Omn1lN9V6bWWNq/M6HQqvmDoCBv1eMldUIJbNgaxrFMxnSM4+TvYWhx+sDWX6oaMPweLt7kaL0Y7vehfy0atl2u921c5mHHXIARGnJr7GTbGejI2IilNJ1xxMtylVz/AjRjhg5PBL3daTQl1rbkc0YyblhmfLBv8uNCPxtzyz5/rBmL65WxryBjVplXtNC22/7aSpztl/yCqNz9plZrmzIWxfWDY80HI7lG12Wosxj5jTJCes9pJ5mhVZdVS2+m3l1hwMfxOf+OCR+Nt3P4E7vsa+A0m0t6MPADBrcwNWVbPdv5XZm50P7U1EwQh7oJmSDg7czgbp1mWPzMdq/aFjHNI0X0bf+PHvaum23M6p6qYuKKUszxnE1+3//XMFWvQOxmZf6LRSue23C8wiexndx+t7b3HZeTpn9uqS6Gw8/PMMopT8tis+Ovi3k2ZFVmlw3ucg12UfOx4H7V9YkziKXtjDBBIRuRF2tsE2OBCRp0WkQUQ2GJYdJSIzRGSb/v+R+nIRkUdFpEJE1onIJwz7XKdvv01Ergvm7QTDbcSW2XxfZx/u1Wez3N9hsxnjqUaO8Jajyp6Upm8gbR0cFBiNetl7IK1sm/i4ToeH3Z3u09LVNziiCeD+c1mSNUFR9iykYTQvKbKRUQEAn/3wMTnLBNaZfbfLswOcYryGREQUf2E/f5zkWJ8BcFnWsrEAZimlTgMwS38NAJcDOE3/dwOAJwAtmABwB4BPAzgPwB2ZgCIsnRbj43uxc1/+UqZ9nX05y5w2m/nH/KEO0vvvZ5xAzdn+ZvpSacuSeb++b1195jNumqVbKf+jYG81B/b79KXSOOfOGcM6GG6uM58R1Kl2H7+LToWdrw0j4DE2HzJ+lm5HKyrGZl1ERFQ8YldzoJSaDyC7/c2VACbof08AcJVh+bNKsxTAESJyPIAvA5ihlGpSSjUDmIHcgCMw767bjbPumIaNu7WZJd3mI7M3N+sr4OYYVud/a80uPDZnaDi9kSMKbWyj6U2lLcfwXu2yvXy2TKb85qzhOjN+9/aGnGVppXJqLAqNir2MAezknP/6+2kAtM7hGXYjQLkXfPY0ThPjuXXZWbkzyAJZQYD+9kQER7xvf9PtrSdBMz8vOwwTEVEcJKXPwXFKqToA0P/PDGp+IgDj0C21+jKr5aGYs1kbhWbjbnfTmBc6A6qR8XM1y8j2pdK4aeKaYctG7refL/0ictq8G/zq1bUFHTvfGP8AUNOU229C63MwXKFfey81B58dNxsA8Pba3WhoNw/4rGbKTprQaw58PNbYy88wXW5WQyAAjj/cfLZty8nRLCdN09z79X8FgLyznRMRERULv4cyNXvKqjzLcw8gcgO0Jkn44Ac/6F/KDInQZkR1vl8hAVum7N+YHTbLyP5ldu4EPJvqhoKZ//rHUs9pCDKD62UG5nTaZFjRgodU9XaA55buxO1vbsApxxxSWAIKUIx9DrLfk5YB95YIq+tjHH0o+8gnHH4QdmfV8FmNVmR33gs/ovVtOKDA4XaJiIiSwOvTrl5vLgT9/8xYZ7UATjZsdxKA3XmW51BKPamUGqOUGjNq1CiPycs6Jtw3YVluKLHPLlX2UqNgPKfZ+RvaenMX+iRfzUHGX+dU4Jrxy1wf20tzHi048rfuwGvm90/TtwAAduwt7tFKwq6SzG4QN7KAsWqtOgyPMOkYnu+nmW/dUYfk1gr406iPiIgoWbwGB28DyIw4dB2AtwzLr9VHLTofQKve7GgagEtF5Ei9I/Kl+rJwGNojO/XLV9di5U5tGNLs5j6ekjAsOMjNqPUNBFe67yQ4+OO0LViwba/rY3tpzpNWKieoiKLPQSH7+ckq89va7W7I1Hyifpteg4PpP7/Qct2wmoPsuSNMrmlm0sEvnXlczro3fvRZbZsDhoYlzZ2rwfoqfu+C0YN/n/GBQy23IyIicit2oxWJyEsAlgA4XURqReR6AOMAfElEtgH4kv4aAN4DUAmgAsA/APwfACilmgDcBWCF/u9OfVkkQu1oKNpISb2podF8ws6Q2s3IO2V9nfdje3gzu1q6cwKWwoMDbwfwkv6wTF7n/XPJ5vX6eJX9G7MaQcjOvxxziPX8BCZ3r7w1B/r/B4zcD98ec9KwdR88+n2ouOdyrL3j0pztnTDWMvjZV6mYuRkmm4iIwmPb50Ap9V8Wqy4x2VYBuNHiOE8DeNpV6gISdgR21h3DK0kKHdPfLbsM8I9eWOX52F4ynfVtvfjt6+uGLVtT48+oSW45qVUJWhhNfqLuczDCLCfvkGWzojx9DrwY6VOfAoYGjj0D4DEAzxqWZYbJHiciY/XXv40gbUREsRF2vrGkethNWFzly3wHhT78wy6sDrLU2Ouhs0eO2lJf2NwBXtMRZHMup8L4OoQeHPjU50BEcgKNzKFGmswDkq+fwOdPH4WzTjgMP//iaYPb//iiUy23f+jb5wymwV2aHW5X4mGEy2GyiYhKVtjPcL9HK4qlzDVdv6sVd0/e5Hq0Iqvjed4/5E950updgR17uQ9Drfoh7GYzfgrj+xD6kKxZ+V6vs30Dw4MAQKtJSCvlulnRYQftj8k//dywZR886n2W23/mw0e7SqeTNJCtYcNki8ixZhsFOaodEVHcxK7PQTEwZr5au/tDv8jZwj7/i8uqAzt2HEregXh0LPYqjO9DR69/nZu98FxzAOD9B/7/9u48PKrqbuD498xkJvu+QAhJSNiXAGENqwgICCiiVETc19etLtUqWi21C2r7Wtv61uWtdrEupVXRqq2trVrf1gVEwV3ZBVwCyC5kO+8f985kMrkzmUlm7kxmfp/nyZOZO2fu/d0z2zn3bG2vYXi6GbVdITk8noXSMlKdHaQMjW+FINQWgYrCwBUTEVw0ZrUTQoh4Jd2KuoFwijlrzRmPfHXnq9zxqjvnaVfGfITqSKO9lTj/z0hHA5L7BCgoKwVpLke7bcYxAu/Tk+bu02sB2q2a/K1ZA1l+whDmDisNGldnhNpykJtuvZJzkgs0TbYQQgibJEXlwLfYuPewvVdQmywuaR9qaLZIKbrC7q5aiei62QMjti//fvquLgz29d/XotHGTENOh+KepaN45ILxAV//gky35fY0l5NzJlXhsKi0/HLpKB69sK7T8YouCTRNthBCCJskR+XAp9zwn427u7y/rxu7Vri/edW7HScSYenO3YrixfmTq6K2785OZeqpGNw4d5B3260LhrF++SzcKQ6OryllYr8i7wUAT3rPZz7cVZEB5taUdnq8ge+xRXBhTpMthBBJS8YcdAOJvppud9QitYMuCzRlaCCDS3MCPua/p66skAxw0dS+3ttOhyInrW2XnMHmwmO9ctMAaGoxulF1ZSC0v2DvsH4lWRE7TrLQWi/RWpdqrV1a695a6we01ru11jO01v3N//Ex44EQQsSQ3SWcpJitSCQ+qRp0XbgX2YMlj9QiaKG6ZFo/JvYrYlSFsWaWZ20Pz0xHXbnqEkrkp40tp/7AUe78+8edP5AQQghhRVoORKR19aptd9CdByTHi3BbDoIV+MN9x1193IAwn9E+Fk/FAHwqBxFsOQhGKcXoSuP4ds8qIYQQIrHJbEVRkOw/1VaDohON1A26LtxidLC6hP8g4mAvT2qKgwUjyzo83uIx5SFG1vqe918jwQ7hVrKEEEKIYGTMQRTITDaJT1oOui6i3Yr87kfi5bl90XC23DYvpLQtNrcc+MpKTeGHC4fZflwhhBCJye5rTklRORCJb8VzH8Y6hG7P/2p/JNPbXXXztBy4YtByALB0fCUAA3tkt3vs7tNr+dVZY+wOSQghRDdl9/XPpBiQLNeUE9/7n+2PdQhJJ9CQgxNG9OKU0b154q0d3m3+rXf3nTmaix96M2qx+Y85iETrodbwo4U1BFqyYVxVASfXlnHVTGP8xNu3HEeay8mdf/+Y44b04Bv3vgrA/OG9uhyLEEKI5GF374ikqBwIISIvUMvBL5bUsnpL4Bkoz55QyczBPXz2E/HQWsccRLhb0enjKwI+5nI6uHPxSO/9vAxjAbYb5w6OaAxCCCGSi90XuZOjW5E0HQgRcZcc0zfgY43NLW3u+170UEqFPfi5s9JcTgCqijJtOqIQQggRWdKtSAjRLYytKgj4WFNz22+y0ZX5vLNjH2C0FER7cFVRlptdBxvISXPx63PGMqI8r9P7ksmHhBBCxJK0HAhhs8cvmRjrECJqwUh7+rT7FppvXTC0zWOeloM0l/EVc/KoMr4zz+heo1BhD34O15OXTuLnS2oBOHZQCQWZ7qgeTwghhIgam5sOkqJyIIsSxUav3LRYhxCSURWdv6ocL365dJT39uKxoa8H4O+ORcM79byzJvRpc7/RbDmY2r+YzSvmMrx3ax4HGtAbSeUFGZw4Qgb+CiGE6P6k5SAKZAr82PjteePonZ8e6zA65H8Vu9aistCvJCvk/f3stJFRu3qfmmL9kZ1bU+q93dDUYpkmFHVVhSEfP9hiX00tRgwup8Obv6ePr2DJuHIun96/TVr7RiAIIYQQoiNJUTkQsdG/RzbfmTck1mGE7bghPdptK8lODev5Ge7ID+e5Z+koZgwu6TDdkcbmgI9NqA5e+O/ZQWvPsuMHeW+nBJrLlNZuRS6f2YIy3CmsOHk4uemuoMeIV9ICKYQQIhZkheQokJaD8MwfXtpxohD5X1zOcDvD3sfcmp4RiqbV+ACDac+sq2w3mBYg3RV63BnuFM6fXNXp2AKpLMz0vpcvPqa63eO/XDrKctEtX49eVMfZEyo5o66CS6YZsw1dPLWav109FQC3RctEXbWRV4N6ZnP2xD7e7a4g/YOm9i+mJDuVi4PMaOQhA36FEEKIwOy+OJUUlQMRHs8CUv5+c+7YsPfl2/WksjCD/n6F11NG9W73nPyMtleWb5gT+XniA3WJ+f5Jw9pNwwmwsf5gu23/G2SV234lWRSH0doQCo32LoRSazH7ztyaUp6/eqq3v78/T8XhewuG8YOTarwVjZx0FwOCVCoundaPlRdP4KHzx7fpguUM0nJQmJXKGzfNZHBpTofnJYQQQojApOUgCqQ7QHh8+6yvvHiC93afwvDnivctP7Zo3a53+UiL/v0XTW17tdkR5F26aHRr5WJQz+BXzX1ZVQA8miwqR1az61h1P/K16rJJ3H/maF6/cUbIcQWjNXhCU0oxqZ/RRcjz38Oq4jN9UAnPm60D3v2Zn4uOrtxrjNV/O6rsRKqF5wcnDYvIfoQQQohEIJWDKJBuReE52tRCRUEGQJsCoScbKwoyQi4Ipvh0PWmxKo/7vTivfPtYb3cXj2ADX2f69MH3T/e9E43pNZeMK2dhbRmAdwabvV83tklbltc6cNqqX75/d6is1I7HFJTlpTNraM+g8YfL088/xaF4+II6Nq+Yy8MX1LVJM2dYT8oLjPOZ2Nc4F8uKjJn1HcWnQ/wA/XLpaLbcNi+ktFam9C8C4Iy6yk7vI571KcyIdQhCCCG6Ic9vuV2SYhE0qyvByWD+8FKeWf8ZYBRMtuw+HNLzGppasOox4unS4nQorpo5gOfe+bzDffkOSG3Rus19CG16Lv/uK0VZbh48ZyxvbN5DXXUh+RkuTh1bzr837GqTblhZLgCjKwtYWFvGipNraGhu4el1O5ncr4gNX7Z2FXrpumneesrUAcXMG17Ks2beAWT6DTB++vJJlrH+wpxb3z/eK2f0Z86wnhz/s1e82x84ewzn/3YNAL86awz/2bib4uxUbv/rh5b71tro9tQ7P51pA41KkVWLhtOhuHBKNbc89R59i7N44Oyx3vUGfI00uyYN7RW8649dlevfnDvOO8vRny+fzGubdttz4A5EajalF645hiT9KhJCCNEFwbrxRkPCVg601mgNDocKOntLd3DxMdXc9/KmDtMp1bYg5zu49C9XTmXwLX8N6XhHm1u8V5NbfHaovV1awB3iZPW+g1ZbtObGuYNZuWY7j76xzTJ9YZaxWNUl0/pyz0sb28Ww7pZZuFIUGe4U79z5b90yCzDGBdzz0kZunj+ElhZNfqabV5dNp2dOGkopnA4naS4nb9w0g4IMN4+t3saRxtYpN319d/4QnEpx0dRqfvfqFuN5W/b4nIv1+Z5gMbe+UoqrjxvQbvuMwa1X82cO6cFM8+q+f+XgyUsnct/Lmxhcmk2K08FNIcwA5eka5nI6SA8wCPz4mlJeXTad0tzg081Gu1vexL5Gi4HTYbxGADW9c6npnRvV44aqJDuVJePKWTq+ay0aKXYs8CCEEEJ0UcL+Wl3/+Hqqb3wOCD61Yzx59puTef3GGe2uruekBZ760bef/QNnj/F2B4K2VzwDFRCt9CvOYoh5Ndm3O01FQQa1FXmsWFhDqsWVaIAfLaxpc7/QZ2Xa5haorchnxcmtaYb2MgqAZXnpvHjtNO8UoNfPGcTKiycwpX8RRVmpvHHjDN64cQa5Ga6A04T2Lc7iJ98YQW66i3zzuKW56e2urpdkp5HidPDydccGHA9QkpN3RWqhAAAVJUlEQVTGz5fUMqwslzsWjWh39dx/LYHfnTeO/9ww3XJf4Xrx2mm88u1jvfdrK/K598zRYRUuS3KMKUmrioOPEwlWMfjDRXWMKM9jSGl0C+l3n96+tSWeOByKFScP97ZECSGEEHayu9E5YVsOVq7ZDsDrm3Z7rw7H0ujKfPoUZvL42u0B0xRmptIjJw2300Fjc2uFJliXcKdDMawsh3d37KcoK5VVl03irhc+5nevbu1woGlBpps9hxrabfcMCD1rQh9Kc9OpLs5kydgK3CkOnrzU6E6z++BRb/q+xZlsrD8EGAtdPb52O9MHGd1eqouzuPPUEVyzch2Hjja1O9bg0mwy3E6+d+JQqoraFmTHVRXw0PnjgdbCbiT1CGOfnq5ptRV5fPz5ASr9+o/3ykujV15kFnyrKsrkaJPx+nd2EbkThpeSl+7y9uMPx8m1ZTQ0tzC+upCnLrPuPuUxtk8+Y/pYTwsbqrQwpokVQgghkk2oY/8iJWFbDjwW3/9azFoOvuXTleTxSyby36eOCJre033GtzvQ5hVzvQVtX2tvPg6AcydVea9qO5SiINPt7UMeqG5w83yjW0qTxYw9fQozSHc7SXc7GWeuBfDPb03jwqlt59VP9SnQ+feFe/ySiVx2bD/vfU9Xm0yLQbwZ7hTev3WOt0tNvMpOM2I/d1IV7906x3su1WaFpl9JaDMljbSYgtRKaoqTRy4c760chUspxdQBxZZjEjpy5+KR3H36qJDS/vG/JnL9nEEdJxRCCCFEp+w51NhxoghK2JYDX3ZXDgb2yOajLw6gFDxy4XhSU0K7MuqpHPj2f1dKMahnDk6HorlFc8ei4YyuzKcg0+2dGeaB/9vcZj++YwN81ZTl8s6OfZwyqozvP/O+5XoGoQ7e9l0UrEVDTloKRVnWU126nA5+YXbR8SjMdJNqseBWvLp65gB65qQxv6btAnH/vHZaWPtZddkk+tzwrPf+SSN7sertnZZpPX3xE9Xfrp7Kll2HYh2GEEIIEdd27v3a1uMlReXg6yhVDgb0yOLjL9ovjuVbKPcv4H3/pGEUZrq59OG1VBVlstmncJRtji3wjA+4bvZA72Onj6vgode2smBkr3aVjdlDe/DBZ/spyTEK557ivUJx28k13vEDnhl2DjcY+eFbEXjl28cy5Y4XaQmxcuB0KP561RTm3PUKLS2add+dFTS9/0DdNd+ZGdJx4kW628l5UVj1+KeLR/LTxSMjvt9oynA7ve+hrhjQIzvo4mtCCCGEsH+9roSsHKzfvrfN/UiNORjeO5f12/cBcN6kKs6cUMmxP3mpXTrPTD9WXTrOrKv0Xi31v3Kfm25UDh48ZywPv7aNS33m+19+4lCunTXQshXim9P7c87EPuRlGINwfVsOThtX4U3nicfTMuE5fr+SLO+2cKZ9LTZbCib2Kwy7+0pnurskou6YD2tvPk7WDhFCCCFsYvdvbkJWDpY//V6b+5FqOfAtzN9ygvV0klP6F9GvJIv3P9sfcD+e/us1vXP50yUTGPfDf7R5vG9xVrv9Ox2K3AzrWYscDuWtGEDHK9+6nIoTRvRiydhyqouzyE5Lodl854XaJx6gMCuVf113LKV5kR8sLOKXDCAWQggh7COzFUWA/5z1vn59zljO/c3qTu23T1Em7+1sW+hPTXFwtKmFYwYUM7lfERea8+KDMT+6lcKsVP58+WT6lWR5Bx9H8gJyaw3TeqdKKcvFup65YjLVHUx96a9CVn0VQgghhIgau/sYJF3lYFBp6H2c/3LlFO+KtkN75XDHKcPbrJoLxoDUL/cfobYi37vtjPGVlGSnMXto4Bl4PAs8efr4d3bKSiueukG4C+rJPO72+PcN06k/cLTjhEIIIYRIeg6buyAnZOUgxRk4E8NZgnpwaQ7XzR7Ij5//iKkDii2n4izLS6fMb357h0MxZ1jPkI7hcCjuWTqqTeWiy3TwbkUitqzeM0IIIYQQVhw2T+6YkJWDQC0HOWkpAVfXDWSsucDThOrCLscVyPF+02N2lcOsAKXY/W4SQgghhBARZXfLQUKWHl0BWg7OqKskKzWFBSNbp9X0LAgG8NPFI7j3jNFtnjOuqoD3b53N1AHF0Qk2ChaN7s05E/twzawBHScWQgghhBBxqyDT3XGiCEqaloMHzxnDsQONlYZvmjeYp8yFp3y7d8wf3svyueG2NsRaaoqT5ScOjXUYQgghhBCiiy6cUm3r8bpXqTdE35zR31v495g+qHVwcEl269SbMwaXMKhnNkvHVwQdyOxx4ZQq73oEQgghhBBCdFZRlptdBxssHzt/chVfNzbbPmFMQlYO+hZnhZzW5XTw16umhpz+pnnW6xsIIYSIDKXUHOBngBP4ldb6thiHJES3N6wsh3d3tF+DydPV2veianVRJpvMBVuDyXQ7OdTQfi2pQT2zueeM0azesodTx5QD0OeGZy338berp/LWtq9oaNbcvOpd/nPDdAoy3Sy+/zXWfdq6qO2y4wexsf4gK9dsJ83l4EcLa7hm5TruWTqKzbsPkZ/h5u5/bmD20J48+O/NAGz44fGkOB28+NGXvLnlK9JcDtwpDp5/7wve3PoVp40t57HVnwLGQrf7vm5k6+7DADx6YR2vbtrNjq++Zv7wUpSCIaU5PLb6UxaM7MUxP36JmrJcVl02iWv/uI5Fo3tTVZTJs+s/w+lQjO1TwOf7j/D5vq+5+anW9bf+8a1jOHikif1HGhnYM9t7wfrdHfv40XMf8ND548OaPCcalI7jpU7HjBmj16xZ06nn+r8Jt9w2r839U+97la8bmvnzFZMDPtf/OUIIEU+UUm9qrcfEOo5IUko5gY+B44DtwGpgidb6fav0nf2dCFRQEYnt4qnVjK7MZ/32fZTlp7N6yx627T5MdloKL35U701XmOnm/rPG8Mjr2/jOvMH865N6rnzsbb57whBKstO47JG1AIwoz2Pdp3vJz3Dxm3PHoYFN9Qe5ZuW6dsc+dUxvrpw5wNud+XBDEx9+foDa8jz+vWE3ZfnprPt0L69v3sOtC4bicjp4f+d+1m77iuqiTHbuO0JeuospA4pITXHS1NxCizbGWR5uaObQ0SZ++NwH3LFoOGc+8AZvbN7Dh9+fwyW/f5MzJ1QyfVAPmls0CmM2w/d27icvw0XvfGO9onk/f4X6A0d5bdkMHA7FV4caSHc7SXM5qT9wlOy0FO8imEebmtm59whVRaGvjbR51yEONzQxtJdxFfyaP7zNE2/t6LCsdaSxmdVb9jClvzH2c8+hBlxORXZaZHpxHGls5q4XPuHKGf1JdyfWIp+d/Y1I2spBKM+VyoEQIp4laOVgArBcaz3bvL8MQGu9wiq9VA7i392n11JZkMnabV9xz0sbGVGeyxXT+9PY3MLOvUc40tjM1j2HueY4YxKNxuYWXnj/Cw4ebWLV2zv46eKRlGSnseqtHbzwwRecOqacXQePUlddSK+8dD7+4gBvbfuKk2rLSE0xCnd7Dzewsf4QoyuNacK/2H+EzbsOURdk5sHG5hYOH20mNyMyhc6vG5rZvOsQGs3n+44wY3DgtY+EiAapHPh5+PWtbNt9mPv+tYkl4ypYcXJNyM998q3tlOVlMK6qoFPHFkIIOyRo5WARMEdrfYF5/0xgvNb6cp80FwEXAVRUVIzeunVr2MeJh8rB7afU8INnP2DO0J4M6ZVDSXYaRVluXvlkFwtHlVFRkMGHnx3g7e17OWN8BUop1n26lyONzQzplUN2movmFo3WmhYNn3x5gE31h9hYf5CFtWU0NreQm+6mKMvNCx98CcB//f5Nlp8whNPGGePsjjQ28/LH9RRnp7J261eU5aWz6u0dZKW6WDZ3EPkZbjbvOki/ktAWEN13uJH6g0fpVxJ6914hRHRI5SCAvYcbyE5zxbz/lhBCRFqCVg6+Acz2qxyM01pfYZW+s78TRxqbuf7x9ew6eJTmFk1pbjoHjjRSnJ2K1kafaafTQXl+OoWZqfTISSUn3YXb6UApULLKpBAiznX2N8L2Acl2DzTLy7B3blghhBBdsh0o97nfG9gZIG2npbmc/Oy02kjvVgghuj1bF0EzB5r9D3A8MARYopSS6X+EEEJ4rAb6K6WqlFJu4DTg6RjHJIQQScPuFZLHARu01pu01g3AY8ACm2MQQggRp7TWTcDlwPPAB8BKrfV7wZ8lhBAiUuzuVlQGfOpzfzsw3uYYhBBCxDGt9XPAc7GOQwghkpHdLQdWI7jajIhWSl2klFqjlFpTX19vkVwIIYQQQggRDXZXDjocaKa1vl9rPUZrPaa4uNjW4IQQQgghhEhmdlcOZKCZEEIIIYQQccrWMQda6yallGegmRN4UAaaCSGEEEIIER9sX+dABpoJIYQQQggRn+zuViSEEEIIIYSIU1I5EEIIIYQQQgCgtNYdp4oRpVQ9sLWTTy8CdkUwnEiJ17ggfmOTuMIjcYUnXuOCjmOr1Fon9bRuCfo7ES3Jdr6QfOcs55v4wjnnTv1GxHXloCuUUmu01mNiHYe/eI0L4jc2iSs8Eld44jUuiO/YEkGy5W+ynS8k3znL+SY+O85ZuhUJIYQQQgghAKkcCCGEEEIIIUyJXDm4P9YBBBCvcUH8xiZxhUfiCk+8xgXxHVsiSLb8TbbzheQ7ZznfxBf1c07YMQdCCCGEEEKI8CRyy4EQQgghhBAiDAlZOVBKzVFKfaSU2qCUusHmY5crpV5USn2glHpPKXWlub1AKfV3pdQn5v98c7tSSv3cjHW9UmpUlONzKqXeUko9Y96vUkq9bsb1B6WU29yeat7fYD7eJ4ox5Sml/qSU+tDMtwnxkF9KqavN1/BdpdSjSqm0WOWXUupBpdSXSql3fbaFnUdKqbPN9J8opc6OUlw/Nl/L9UqpJ5VSeT6PLTPj+kgpNdtne0Q/s1Zx+Tx2rVJKK6WKzPsxzS9z+xXm+b+nlLrDZ7st+ZWMunseKqW2KKXeUUq9rZRaY26L2HeCUmq0uf8N5nOVzecX1e+8QOcX6BgxOt/lSqkd5mv8tlJqrs9jYX03qDj4rfc734iVlbrDaxzkfOPzNdZaJ9Qf4AQ2AtWAG1gHDLHx+KXAKPN2NvAxMAS4A7jB3H4DcLt5ey7wF0ABdcDrUY7vGuAR4Bnz/krgNPP2vcAl5u1LgXvN26cBf4hiTL8FLjBvu4G8WOcXUAZsBtJ98umcWOUXMBUYBbzrsy2sPAIKgE3m/3zzdn4U4poFpJi3b/eJa4j5eUwFqszPqTMan1mruMzt5cDzGPPiF8VJfh0LvACkmvdL7M6vZPtLhDwEtnjewz7bIvadALwBTDCf8xfgeJvPL6rfeYHOL9AxYnS+y4FrLdKG/d1AHPzW+51DRMpK3eU1DnK+cfka2/ZBt+vPfCM873N/GbAshvE8BRwHfASU+rxJPjJv3wcs8UnvTReFWHoD/wCmA8+YH5hdtBbkvHmHUYCaYN5OMdOpKMSUg1EIV37bY5pfGJWDT80vnBQzv2bHMr+APrT94Qgrj4AlwH0+29uki1Rcfo8tBB42b7f5LHryLFqfWau4gD8BI/ApWMU6vzC+0GdapLM1v5LpLxHyEOvKQUS+E8zHPvTZ3iadjecYle+8YOcX6BgxOt/lWBccw/puIA5+60M4906Vlbrba2xxvnH5GidityJPoc5ju7nNdmbTTS3wOtBDa/0ZgPm/xExmZ7x3Ad8GWsz7hcBerXWTxbG9cZmP7zPTR1o1UA/8WhndnX6llMokxvmltd4B/ATYBnyGcf5vEvv88hVuHsXis3EexhWbmMellDoR2KG1Xuf3UKzzawAwxWzufVkpNTZO4kpkiZCHGvibUupNpdRF5rZIfSeUmbf9t8eaHecX6BixcrnZjeZBn+4v4Z5vPPzWB9TFslK3e439zhfi8DVOxMqBVb9IbXsQSmUBjwNXaa33B0tqsS3i8Sql5gNfaq3fDPHYduVjCkZT6j1a61rgEEYzXyB25Vc+sACjOa8XkAkcH+TYcfG+MwWKxdYYlVI3AU3Aw7GOSymVAdwE3GL1cKziMqVgNIfXAdcBK82+sbGOK5ElQh5O0lqPwvheukwpNTVI2nDfS90tfxL1/O4B+gIjMS5S/be5PZLnG9O8iEBZqVu9xhbnG5evcSJWDrZj9Cv26A3stDMApZQL48V/WGv9hLn5C6VUqfl4KfClud2ueCcBJyqltgCPYXQtugvIU0qlWBzbG5f5eC6wJwpxbQe2a609Neg/YVQWYp1fM4HNWut6rXUj8AQwkdjnl69w88i2z4Y5KGw+sFSbbZkxjqsvRkVvnfkZ6A2sVUr1jHFcmMd5QhvewGjZK4qDuBJZt89DrfVO8/+XwJPAOCL3nbDdvO2/PdbsOL9Ax7Cd1voLrXWz1roF+F+M1xjCP99dxNdvF+bxIlFW6javsdX5xutrnIiVg9VAf3PUthtj8MXTdh3cvOL3APCB1vpOn4eeBs42b5+N0d/Ms/0scyR+HbDP09wVSVrrZVrr3lrrPhh58k+t9VLgRWBRgLg88S4y00e81q21/hz4VCk10Nw0A3ifGOcXRneiOqVUhvmaeuKKaX75CTePngdmKaXyzZaRWea2iFJKzQGuB07UWh/2i/c0c+aEKqA/xoCxqH9mtdbvaK1LtNZ9zM/AdozBYZ8T4/wCVmFU1lFKDcAYZLaLGOZXEujWeaiUylRKZXtuY7w33yVC3wnmYweUUnXm999ZPvuKJTvOL9AxbOcpwJoWYrzGEOZ3g/lbFE+/XZEsK3WL1zjQ+cbtaxzpQRbx8Icxqv1jjBHdN9l87MkYzTXrgbfNv7kY/bv+AXxi/i8w0yvgf8xY3wHG2BDjNFpnK6o233AbgD/SOmNKmnl/g/l4dRTjGQmsMfNsFUYXi5jnF/A94EPzw/oQxqwBMckv4FGMJsdGjILt+Z3JI4wxABvMv3OjFNcGjP6Nnvf/vT7pbzLj+gif2U8i/Zm1isvv8S20DkiOdX65gd+b77O1wHS78ysZ/7pzHprfQ+vMv/c88UfyOwEYY74nNwJ3Y/Mg1Wh/5wU6v0DHiNH5PmSez3qMAl6pT/qwvhuIg996v/ONWFmpO7zGQc43Ll9jWSFZCCGEEEIIASRmtyIhhBBCCCFEJ0jlQAghhBBCCAFI5UAIIYQQQghhksqBEEIIIYQQApDKgRBCCCGEEMIklQMhhBBCCCEEIJUDIYQQQgghhEkqB0IIIYQQQggA/h9OI8M9DW4WsAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_training(frame_idx, rewards, losses):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, np.mean(rewards[-10:])))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.title('loss')\n",
    "    plt.plot(losses)\n",
    "    plt.show()\n",
    "\n",
    "plot_training(i, all_rewards, losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "action_dim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
